remove self managed protocols as it has been replaced with official oai spec (#17711)
This commit is contained in:
Simo Lin
@@ -1,7 +0,0 @@
|
||||
//! Builders for Chat Completion API response types
|
||||
|
||||
pub mod response;
|
||||
pub mod stream_response;
|
||||
|
||||
pub use response::ChatCompletionResponseBuilder;
|
||||
pub use stream_response::ChatCompletionStreamResponseBuilder;
|
||||
@@ -1,218 +0,0 @@
|
||||
//! Builder for ChatCompletionResponse
|
||||
//!
|
||||
//! Provides an ergonomic fluent API for constructing chat completion responses.
|
||||
|
||||
use crate::protocols::{chat::*, common::Usage};
|
||||
|
||||
/// Builder for ChatCompletionResponse
|
||||
///
|
||||
/// Provides a fluent interface for constructing chat completion responses with sensible defaults.
|
||||
#[must_use = "Builder does nothing until .build() is called"]
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct ChatCompletionResponseBuilder {
|
||||
id: String,
|
||||
object: String,
|
||||
created: u64,
|
||||
model: String,
|
||||
choices: Vec<ChatChoice>,
|
||||
usage: Option<Usage>,
|
||||
system_fingerprint: Option<String>,
|
||||
}
|
||||
|
||||
impl ChatCompletionResponseBuilder {
|
||||
/// Create a new builder with required fields
|
||||
///
|
||||
/// # Arguments
|
||||
/// - `id`: Completion ID (e.g., "chatcmpl_abc123")
|
||||
/// - `model`: Model name used for generation
|
||||
pub fn new(id: impl Into<String>, model: impl Into<String>) -> Self {
|
||||
Self {
|
||||
id: id.into(),
|
||||
object: "chat.completion".to_string(),
|
||||
created: chrono::Utc::now().timestamp() as u64,
|
||||
model: model.into(),
|
||||
choices: Vec::new(),
|
||||
usage: None,
|
||||
system_fingerprint: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Copy common fields from a ChatCompletionRequest
|
||||
///
|
||||
/// This populates the model field from the request.
|
||||
pub fn copy_from_request(mut self, request: &ChatCompletionRequest) -> Self {
|
||||
self.model = request.model.clone();
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the object type (default: "chat.completion")
|
||||
pub fn object(mut self, object: impl Into<String>) -> Self {
|
||||
self.object = object.into();
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the creation timestamp (default: current time)
|
||||
pub fn created(mut self, timestamp: u64) -> Self {
|
||||
self.created = timestamp;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the choices
|
||||
pub fn choices(mut self, choices: Vec<ChatChoice>) -> Self {
|
||||
self.choices = choices;
|
||||
self
|
||||
}
|
||||
|
||||
/// Add a single choice
|
||||
pub fn add_choice(mut self, choice: ChatChoice) -> Self {
|
||||
self.choices.push(choice);
|
||||
self
|
||||
}
|
||||
|
||||
/// Set usage information
|
||||
pub fn usage(mut self, usage: Usage) -> Self {
|
||||
self.usage = Some(usage);
|
||||
self
|
||||
}
|
||||
|
||||
/// Set usage if provided (handles Option)
|
||||
pub fn maybe_usage(mut self, usage: Option<Usage>) -> Self {
|
||||
if let Some(u) = usage {
|
||||
self.usage = Some(u);
|
||||
}
|
||||
self
|
||||
}
|
||||
|
||||
/// Set system fingerprint if provided (handles Option)
|
||||
pub fn maybe_system_fingerprint(mut self, fingerprint: Option<impl Into<String>>) -> Self {
|
||||
if let Some(fp) = fingerprint {
|
||||
self.system_fingerprint = Some(fp.into());
|
||||
}
|
||||
self
|
||||
}
|
||||
|
||||
/// Build the ChatCompletionResponse
|
||||
pub fn build(self) -> ChatCompletionResponse {
|
||||
ChatCompletionResponse {
|
||||
id: self.id,
|
||||
object: self.object,
|
||||
created: self.created,
|
||||
model: self.model,
|
||||
choices: self.choices,
|
||||
usage: self.usage,
|
||||
system_fingerprint: self.system_fingerprint,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Tests
|
||||
// ============================================================================
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_build_minimal() {
|
||||
let response = ChatCompletionResponse::builder("chatcmpl_123", "gpt-4").build();
|
||||
|
||||
assert_eq!(response.id, "chatcmpl_123");
|
||||
assert_eq!(response.model, "gpt-4");
|
||||
assert_eq!(response.object, "chat.completion");
|
||||
assert!(response.choices.is_empty());
|
||||
assert!(response.usage.is_none());
|
||||
assert!(response.system_fingerprint.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_build_complete() {
|
||||
let choice = ChatChoice {
|
||||
index: 0,
|
||||
message: ChatCompletionMessage {
|
||||
role: "assistant".to_string(),
|
||||
content: Some("Hello!".to_string()),
|
||||
tool_calls: None,
|
||||
reasoning_content: None,
|
||||
},
|
||||
logprobs: None,
|
||||
finish_reason: Some("stop".to_string()),
|
||||
matched_stop: None,
|
||||
hidden_states: None,
|
||||
};
|
||||
|
||||
let usage = Usage {
|
||||
prompt_tokens: 10,
|
||||
completion_tokens: 20,
|
||||
total_tokens: 30,
|
||||
completion_tokens_details: None,
|
||||
};
|
||||
|
||||
let response = ChatCompletionResponse::builder("chatcmpl_456", "gpt-4")
|
||||
.choices(vec![choice.clone()])
|
||||
.maybe_usage(Some(usage))
|
||||
.maybe_system_fingerprint(Some("fp_123abc"))
|
||||
.build();
|
||||
|
||||
assert_eq!(response.id, "chatcmpl_456");
|
||||
assert_eq!(response.choices.len(), 1);
|
||||
assert_eq!(response.choices[0].index, 0);
|
||||
assert!(response.usage.is_some());
|
||||
assert_eq!(response.system_fingerprint.as_ref().unwrap(), "fp_123abc");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_add_multiple_choices() {
|
||||
let choice1 = ChatChoice {
|
||||
index: 0,
|
||||
message: ChatCompletionMessage {
|
||||
role: "assistant".to_string(),
|
||||
content: Some("Option 1".to_string()),
|
||||
tool_calls: None,
|
||||
reasoning_content: None,
|
||||
},
|
||||
logprobs: None,
|
||||
finish_reason: Some("stop".to_string()),
|
||||
matched_stop: None,
|
||||
hidden_states: None,
|
||||
};
|
||||
|
||||
let choice2 = ChatChoice {
|
||||
index: 1,
|
||||
message: ChatCompletionMessage {
|
||||
role: "assistant".to_string(),
|
||||
content: Some("Option 2".to_string()),
|
||||
tool_calls: None,
|
||||
reasoning_content: None,
|
||||
},
|
||||
logprobs: None,
|
||||
finish_reason: Some("stop".to_string()),
|
||||
matched_stop: None,
|
||||
hidden_states: None,
|
||||
};
|
||||
|
||||
let response = ChatCompletionResponse::builder("chatcmpl_789", "gpt-4")
|
||||
.add_choice(choice1)
|
||||
.add_choice(choice2)
|
||||
.build();
|
||||
|
||||
assert_eq!(response.choices.len(), 2);
|
||||
assert_eq!(response.choices[0].index, 0);
|
||||
assert_eq!(response.choices[1].index, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_copy_from_request() {
|
||||
let request = ChatCompletionRequest {
|
||||
messages: vec![],
|
||||
model: "gpt-3.5-turbo".to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let response = ChatCompletionResponse::builder("chatcmpl_101", "gpt-4")
|
||||
.copy_from_request(&request)
|
||||
.build();
|
||||
|
||||
assert_eq!(response.model, "gpt-3.5-turbo"); // Copied from request
|
||||
}
|
||||
}
|
||||
@@ -1,421 +0,0 @@
|
||||
//! Builder for ChatCompletionStreamResponse
|
||||
//!
|
||||
//! Provides an ergonomic fluent API for constructing streaming chat completion responses.
|
||||
|
||||
use std::borrow::Cow;
|
||||
|
||||
use crate::protocols::{
|
||||
chat::*,
|
||||
common::{FunctionCallDelta, ToolCallDelta, Usage},
|
||||
};
|
||||
|
||||
/// Builder for ChatCompletionStreamResponse
|
||||
///
|
||||
/// Provides a fluent interface for constructing streaming chat completion chunks with sensible defaults.
|
||||
#[must_use = "Builder does nothing until .build() is called"]
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct ChatCompletionStreamResponseBuilder {
|
||||
id: String,
|
||||
object: String,
|
||||
created: u64,
|
||||
model: String,
|
||||
choices: Vec<ChatStreamChoice>,
|
||||
usage: Option<Usage>,
|
||||
system_fingerprint: Option<String>,
|
||||
}
|
||||
|
||||
impl ChatCompletionStreamResponseBuilder {
|
||||
/// Create a new builder with required fields
|
||||
///
|
||||
/// # Arguments
|
||||
/// - `id`: Completion ID (e.g., "chatcmpl_abc123")
|
||||
/// - `model`: Model name used for generation
|
||||
pub fn new(id: impl Into<String>, model: impl Into<String>) -> Self {
|
||||
Self {
|
||||
id: id.into(),
|
||||
object: "chat.completion.chunk".to_string(),
|
||||
created: chrono::Utc::now().timestamp() as u64,
|
||||
model: model.into(),
|
||||
choices: Vec::new(),
|
||||
usage: None,
|
||||
system_fingerprint: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Copy common fields from a ChatCompletionRequest
|
||||
///
|
||||
/// This populates the model field from the request.
|
||||
pub fn copy_from_request(mut self, request: &ChatCompletionRequest) -> Self {
|
||||
self.model = request.model.clone();
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the object type (default: "chat.completion.chunk")
|
||||
pub fn object(mut self, object: impl Into<String>) -> Self {
|
||||
self.object = object.into();
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the creation timestamp (default: current time)
|
||||
pub fn created(mut self, timestamp: u64) -> Self {
|
||||
self.created = timestamp;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the choices
|
||||
pub fn choices(mut self, choices: Vec<ChatStreamChoice>) -> Self {
|
||||
self.choices = choices;
|
||||
self
|
||||
}
|
||||
|
||||
/// Add a single choice (delta)
|
||||
pub fn add_choice(mut self, choice: ChatStreamChoice) -> Self {
|
||||
self.choices.push(choice);
|
||||
self
|
||||
}
|
||||
|
||||
/// Set usage information (typically sent in final chunk)
|
||||
pub fn usage(mut self, usage: Usage) -> Self {
|
||||
self.usage = Some(usage);
|
||||
self
|
||||
}
|
||||
|
||||
/// Set system fingerprint if provided (handles Option)
|
||||
pub fn maybe_system_fingerprint(mut self, fingerprint: Option<impl Into<String>>) -> Self {
|
||||
if let Some(fp) = fingerprint {
|
||||
self.system_fingerprint = Some(fp.into());
|
||||
}
|
||||
self
|
||||
}
|
||||
|
||||
/// Set usage if provided (handles Option)
|
||||
pub fn maybe_usage(mut self, usage: Option<Usage>) -> Self {
|
||||
if let Some(u) = usage {
|
||||
self.usage = Some(u);
|
||||
}
|
||||
self
|
||||
}
|
||||
|
||||
/// Add a choice delta that sets `role` and `content`
|
||||
pub fn add_choice_content(
|
||||
mut self,
|
||||
index: u32,
|
||||
role: impl Into<String>,
|
||||
content: impl Into<String>,
|
||||
) -> Self {
|
||||
self.choices.push(ChatStreamChoice {
|
||||
index,
|
||||
delta: ChatMessageDelta {
|
||||
role: Some(role.into()),
|
||||
content: Some(content.into()),
|
||||
tool_calls: None,
|
||||
reasoning_content: None,
|
||||
},
|
||||
logprobs: None,
|
||||
finish_reason: None,
|
||||
matched_stop: None,
|
||||
});
|
||||
self
|
||||
}
|
||||
|
||||
/// Add a choice delta that sets `role`, `content`, and `logprobs`
|
||||
pub fn add_choice_content_with_logprobs(
|
||||
mut self,
|
||||
index: u32,
|
||||
role: impl Into<String>,
|
||||
content: impl Into<String>,
|
||||
logprobs: Option<crate::protocols::common::ChatLogProbs>,
|
||||
) -> Self {
|
||||
self.choices.push(ChatStreamChoice {
|
||||
index,
|
||||
delta: ChatMessageDelta {
|
||||
role: Some(role.into()),
|
||||
content: Some(content.into()),
|
||||
tool_calls: None,
|
||||
reasoning_content: None,
|
||||
},
|
||||
logprobs,
|
||||
finish_reason: None,
|
||||
matched_stop: None,
|
||||
});
|
||||
self
|
||||
}
|
||||
|
||||
/// Add a choice delta that only sets `role`
|
||||
pub fn add_choice_role(mut self, index: u32, role: impl Into<String>) -> Self {
|
||||
self.choices.push(ChatStreamChoice {
|
||||
index,
|
||||
delta: ChatMessageDelta {
|
||||
role: Some(role.into()),
|
||||
content: None,
|
||||
tool_calls: None,
|
||||
reasoning_content: None,
|
||||
},
|
||||
logprobs: None,
|
||||
finish_reason: None,
|
||||
matched_stop: None,
|
||||
});
|
||||
self
|
||||
}
|
||||
|
||||
/// Add a choice delta that appends a tool-call *arguments delta*
|
||||
/// Uses `Cow` so you can pass `&str` or `String` without extra clones
|
||||
pub fn add_choice_tool_args(
|
||||
mut self,
|
||||
index: u32,
|
||||
args_delta: impl Into<Cow<'static, str>>,
|
||||
) -> Self {
|
||||
self.choices.push(ChatStreamChoice {
|
||||
index,
|
||||
delta: ChatMessageDelta {
|
||||
role: Some("assistant".to_string()),
|
||||
content: None,
|
||||
tool_calls: Some(vec![ToolCallDelta {
|
||||
index: 0,
|
||||
id: None,
|
||||
tool_type: None,
|
||||
function: Some(FunctionCallDelta {
|
||||
name: None,
|
||||
arguments: Some(args_delta.into().into_owned()),
|
||||
}),
|
||||
}]),
|
||||
reasoning_content: None,
|
||||
},
|
||||
logprobs: None,
|
||||
finish_reason: None,
|
||||
matched_stop: None,
|
||||
});
|
||||
self
|
||||
}
|
||||
|
||||
/// Add a choice delta that sets reasoning content (for models that stream reasoning)
|
||||
pub fn add_choice_reasoning(mut self, index: u32, reasoning: impl Into<String>) -> Self {
|
||||
self.choices.push(ChatStreamChoice {
|
||||
index,
|
||||
delta: ChatMessageDelta {
|
||||
role: Some("assistant".to_string()),
|
||||
content: None,
|
||||
tool_calls: None,
|
||||
reasoning_content: Some(reasoning.into()),
|
||||
},
|
||||
logprobs: None,
|
||||
finish_reason: None,
|
||||
matched_stop: None,
|
||||
});
|
||||
self
|
||||
}
|
||||
|
||||
/// Add a choice delta for tool call with function name and ID
|
||||
pub fn add_choice_tool_name(
|
||||
mut self,
|
||||
index: u32,
|
||||
tool_call_id: impl Into<String>,
|
||||
function_name: impl Into<String>,
|
||||
) -> Self {
|
||||
self.choices.push(ChatStreamChoice {
|
||||
index,
|
||||
delta: ChatMessageDelta {
|
||||
role: Some("assistant".to_string()),
|
||||
content: None,
|
||||
tool_calls: Some(vec![ToolCallDelta {
|
||||
index: 0,
|
||||
id: Some(tool_call_id.into()),
|
||||
tool_type: Some("function".to_string()),
|
||||
function: Some(FunctionCallDelta {
|
||||
name: Some(function_name.into()),
|
||||
arguments: None,
|
||||
}),
|
||||
}]),
|
||||
reasoning_content: None,
|
||||
},
|
||||
logprobs: None,
|
||||
finish_reason: None,
|
||||
matched_stop: None,
|
||||
});
|
||||
self
|
||||
}
|
||||
|
||||
/// Add a choice delta with a pre-constructed ToolCallDelta
|
||||
/// Useful when you already have a ToolCallDelta object to emit
|
||||
pub fn add_choice_tool_call_delta(
|
||||
mut self,
|
||||
index: u32,
|
||||
tool_call_delta: ToolCallDelta,
|
||||
) -> Self {
|
||||
self.choices.push(ChatStreamChoice {
|
||||
index,
|
||||
delta: ChatMessageDelta {
|
||||
role: Some("assistant".to_string()),
|
||||
content: None,
|
||||
tool_calls: Some(vec![tool_call_delta]),
|
||||
reasoning_content: None,
|
||||
},
|
||||
logprobs: None,
|
||||
finish_reason: None,
|
||||
matched_stop: None,
|
||||
});
|
||||
self
|
||||
}
|
||||
|
||||
/// Add a choice with finish_reason (final chunk)
|
||||
/// This is used for the last chunk in a stream to signal completion
|
||||
pub fn add_choice_finish_reason(
|
||||
mut self,
|
||||
index: u32,
|
||||
finish_reason: impl Into<String>,
|
||||
matched_stop: Option<serde_json::Value>,
|
||||
) -> Self {
|
||||
self.choices.push(ChatStreamChoice {
|
||||
index,
|
||||
delta: ChatMessageDelta {
|
||||
role: None,
|
||||
content: None,
|
||||
tool_calls: None,
|
||||
reasoning_content: None,
|
||||
},
|
||||
logprobs: None,
|
||||
finish_reason: Some(finish_reason.into()),
|
||||
matched_stop,
|
||||
});
|
||||
self
|
||||
}
|
||||
|
||||
/// Build the ChatCompletionStreamResponse
|
||||
pub fn build(self) -> ChatCompletionStreamResponse {
|
||||
ChatCompletionStreamResponse {
|
||||
id: self.id,
|
||||
object: self.object,
|
||||
created: self.created,
|
||||
model: self.model,
|
||||
system_fingerprint: self.system_fingerprint,
|
||||
choices: self.choices,
|
||||
usage: self.usage,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Tests
|
||||
// ============================================================================
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_build_minimal() {
|
||||
let chunk = ChatCompletionStreamResponseBuilder::new("chatcmpl_123", "gpt-4").build();
|
||||
|
||||
assert_eq!(chunk.id, "chatcmpl_123");
|
||||
assert_eq!(chunk.model, "gpt-4");
|
||||
assert_eq!(chunk.object, "chat.completion.chunk");
|
||||
assert!(chunk.choices.is_empty());
|
||||
assert!(chunk.usage.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_with_content_delta() {
|
||||
let chunk = ChatCompletionStreamResponseBuilder::new("chatcmpl_456", "gpt-4")
|
||||
.add_choice_content(0, "assistant", "Hello")
|
||||
.build();
|
||||
|
||||
assert_eq!(chunk.choices.len(), 1);
|
||||
assert_eq!(chunk.choices[0].index, 0);
|
||||
assert_eq!(chunk.choices[0].delta.content.as_ref().unwrap(), "Hello");
|
||||
assert_eq!(chunk.choices[0].delta.role.as_ref().unwrap(), "assistant");
|
||||
assert!(chunk.choices[0].finish_reason.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_with_role_delta() {
|
||||
let chunk = ChatCompletionStreamResponseBuilder::new("chatcmpl_789", "gpt-4")
|
||||
.add_choice_role(0, "assistant")
|
||||
.build();
|
||||
|
||||
assert_eq!(chunk.choices.len(), 1);
|
||||
assert_eq!(chunk.choices[0].delta.role.as_ref().unwrap(), "assistant");
|
||||
assert!(chunk.choices[0].delta.content.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_with_finish_reason() {
|
||||
let chunk = ChatCompletionStreamResponseBuilder::new("chatcmpl_101", "gpt-4")
|
||||
.add_choice_finish_reason(0, "stop", None)
|
||||
.build();
|
||||
|
||||
assert_eq!(chunk.choices.len(), 1);
|
||||
assert_eq!(chunk.choices[0].finish_reason.as_ref().unwrap(), "stop");
|
||||
assert!(chunk.choices[0].delta.content.is_none());
|
||||
assert!(chunk.choices[0].delta.role.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_multiple_deltas() {
|
||||
let chunk = ChatCompletionStreamResponseBuilder::new("chatcmpl_202", "gpt-4")
|
||||
.add_choice_role(0, "assistant")
|
||||
.add_choice_content(0, "assistant", "Hello")
|
||||
.add_choice_content(0, "assistant", " world")
|
||||
.add_choice_finish_reason(0, "stop", None)
|
||||
.build();
|
||||
|
||||
assert_eq!(chunk.choices.len(), 4); // role + 2 content + finish
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_with_usage() {
|
||||
let usage = Usage {
|
||||
prompt_tokens: 10,
|
||||
completion_tokens: 20,
|
||||
total_tokens: 30,
|
||||
completion_tokens_details: None,
|
||||
};
|
||||
|
||||
let chunk = ChatCompletionStreamResponseBuilder::new("chatcmpl_303", "gpt-4")
|
||||
.add_choice_finish_reason(0, "stop", None)
|
||||
.usage(usage)
|
||||
.build();
|
||||
|
||||
assert!(chunk.usage.is_some());
|
||||
assert_eq!(chunk.usage.as_ref().unwrap().total_tokens, 30);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_copy_from_request() {
|
||||
let request = ChatCompletionRequest {
|
||||
messages: vec![],
|
||||
model: "gpt-3.5-turbo".to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let chunk = ChatCompletionStreamResponseBuilder::new("chatcmpl_404", "gpt-4")
|
||||
.copy_from_request(&request)
|
||||
.add_choice_content(0, "assistant", "test")
|
||||
.build();
|
||||
|
||||
assert_eq!(chunk.model, "gpt-3.5-turbo"); // Copied from request
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_add_choice_explicit() {
|
||||
let choice = ChatStreamChoice {
|
||||
index: 0,
|
||||
delta: ChatMessageDelta {
|
||||
role: Some("assistant".to_string()),
|
||||
content: Some("Hello".to_string()),
|
||||
tool_calls: None,
|
||||
reasoning_content: None,
|
||||
},
|
||||
logprobs: None,
|
||||
finish_reason: None,
|
||||
matched_stop: None,
|
||||
};
|
||||
|
||||
let chunk = ChatCompletionStreamResponseBuilder::new("chatcmpl_505", "gpt-4")
|
||||
.add_choice(choice)
|
||||
.build();
|
||||
|
||||
assert_eq!(chunk.choices.len(), 1);
|
||||
assert_eq!(chunk.choices[0].delta.role.as_ref().unwrap(), "assistant");
|
||||
assert_eq!(chunk.choices[0].delta.content.as_ref().unwrap(), "Hello");
|
||||
}
|
||||
}
|
||||
@@ -1,27 +0,0 @@
|
||||
//! Builder patterns for protocol response types
|
||||
//!
|
||||
//! This module provides ergonomic builders for response types with many optional fields.
|
||||
//! Builders help avoid telescoping constructors and make construction intent clear.
|
||||
//!
|
||||
//! # Organization
|
||||
//!
|
||||
//! Builders are organized by API:
|
||||
//! - `chat/` - Chat Completion API builders (response, stream_response)
|
||||
//! - `responses/` - Responses API builder (response)
|
||||
//!
|
||||
//! # Optional Fields
|
||||
//!
|
||||
//! For optional fields, builders provide `maybe_*` methods that handle `Option<T>` directly:
|
||||
//! ```ignore
|
||||
//! builder
|
||||
//! .field(value)
|
||||
//! .maybe_optional_field(optional_value) // Accepts Option<T>
|
||||
//! .build()
|
||||
//! ```
|
||||
|
||||
pub mod chat;
|
||||
pub mod responses;
|
||||
|
||||
// Re-export all builders for convenient access
|
||||
pub use chat::{ChatCompletionResponseBuilder, ChatCompletionStreamResponseBuilder};
|
||||
pub use responses::ResponsesResponseBuilder;
|
||||
@@ -1,5 +0,0 @@
|
||||
//! Builders for Responses API response types
|
||||
|
||||
pub mod response;
|
||||
|
||||
pub use response::ResponsesResponseBuilder;
|
||||
@@ -1,411 +0,0 @@
|
||||
//! Builder for ResponsesResponse
|
||||
//!
|
||||
//! Provides an ergonomic fluent API for constructing ResponsesResponse instances.
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::protocols::responses::*;
|
||||
|
||||
/// Builder for ResponsesResponse
|
||||
///
|
||||
/// Provides a fluent interface for constructing responses with sensible defaults.
|
||||
#[must_use = "Builder does nothing until .build() is called"]
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct ResponsesResponseBuilder {
|
||||
id: String,
|
||||
object: String,
|
||||
created_at: i64,
|
||||
status: ResponseStatus,
|
||||
error: Option<Value>,
|
||||
incomplete_details: Option<Value>,
|
||||
instructions: Option<String>,
|
||||
max_output_tokens: Option<u32>,
|
||||
model: String,
|
||||
output: Vec<ResponseOutputItem>,
|
||||
parallel_tool_calls: bool,
|
||||
previous_response_id: Option<String>,
|
||||
reasoning: Option<ReasoningInfo>,
|
||||
store: bool,
|
||||
temperature: Option<f32>,
|
||||
text: Option<TextConfig>,
|
||||
tool_choice: String,
|
||||
tools: Vec<ResponseTool>,
|
||||
top_p: Option<f32>,
|
||||
truncation: Option<String>,
|
||||
usage: Option<ResponsesUsage>,
|
||||
user: Option<String>,
|
||||
safety_identifier: Option<String>,
|
||||
metadata: HashMap<String, Value>,
|
||||
}
|
||||
|
||||
impl ResponsesResponseBuilder {
|
||||
/// Create a new builder with required fields
|
||||
///
|
||||
/// # Arguments
|
||||
/// - `id`: Response ID (e.g., "resp_abc123")
|
||||
/// - `model`: Model name used for generation
|
||||
pub fn new(id: impl Into<String>, model: impl Into<String>) -> Self {
|
||||
Self {
|
||||
id: id.into(),
|
||||
object: "response".to_string(),
|
||||
created_at: chrono::Utc::now().timestamp(),
|
||||
status: ResponseStatus::InProgress,
|
||||
error: None,
|
||||
incomplete_details: None,
|
||||
instructions: None,
|
||||
max_output_tokens: None,
|
||||
model: model.into(),
|
||||
output: Vec::new(),
|
||||
parallel_tool_calls: true,
|
||||
previous_response_id: None,
|
||||
reasoning: None,
|
||||
store: true,
|
||||
temperature: None,
|
||||
text: None,
|
||||
tool_choice: "auto".to_string(),
|
||||
tools: Vec::new(),
|
||||
top_p: None,
|
||||
truncation: None,
|
||||
usage: None,
|
||||
user: None,
|
||||
safety_identifier: None,
|
||||
metadata: HashMap::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Copy common fields from a ResponsesRequest
|
||||
///
|
||||
/// This populates fields like instructions, max_output_tokens, temperature, etc.
|
||||
/// from the original request, making it easy to construct a response that mirrors
|
||||
/// the request parameters.
|
||||
///
|
||||
/// Note: `safety_identifier` is intentionally NOT copied as it is for content moderation
|
||||
/// and should be set independently from the request's `user` field (which is for billing/tracking).
|
||||
pub fn copy_from_request(mut self, request: &ResponsesRequest) -> Self {
|
||||
self.instructions = request.instructions.clone();
|
||||
self.max_output_tokens = request.max_output_tokens;
|
||||
self.parallel_tool_calls = request.parallel_tool_calls.unwrap_or(true);
|
||||
self.previous_response_id = request.previous_response_id.clone();
|
||||
self.store = request.store.unwrap_or(true);
|
||||
self.temperature = request.temperature;
|
||||
self.tool_choice = if let Some(ref tc) = request.tool_choice {
|
||||
serde_json::to_string(tc).unwrap_or_else(|_| "auto".to_string())
|
||||
} else {
|
||||
"auto".to_string()
|
||||
};
|
||||
self.tools = request.tools.clone().unwrap_or_default();
|
||||
self.top_p = request.top_p;
|
||||
self.user = request.user.clone();
|
||||
self.metadata = request.metadata.clone().unwrap_or_default();
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the object type (default: "response")
|
||||
pub fn object(mut self, object: impl Into<String>) -> Self {
|
||||
self.object = object.into();
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the creation timestamp (default: current time)
|
||||
pub fn created_at(mut self, timestamp: i64) -> Self {
|
||||
self.created_at = timestamp;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the response status
|
||||
pub fn status(mut self, status: ResponseStatus) -> Self {
|
||||
self.status = status;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set error information (if status is failed)
|
||||
pub fn error(mut self, error: Value) -> Self {
|
||||
self.error = Some(error);
|
||||
self
|
||||
}
|
||||
|
||||
/// Set incomplete details (if response was truncated)
|
||||
pub fn incomplete_details(mut self, details: Value) -> Self {
|
||||
self.incomplete_details = Some(details);
|
||||
self
|
||||
}
|
||||
|
||||
/// Set system instructions
|
||||
pub fn instructions(mut self, instructions: impl Into<String>) -> Self {
|
||||
self.instructions = Some(instructions.into());
|
||||
self
|
||||
}
|
||||
|
||||
/// Set max output tokens
|
||||
pub fn max_output_tokens(mut self, tokens: u32) -> Self {
|
||||
self.max_output_tokens = Some(tokens);
|
||||
self
|
||||
}
|
||||
|
||||
/// Set output items
|
||||
pub fn output(mut self, output: Vec<ResponseOutputItem>) -> Self {
|
||||
self.output = output;
|
||||
self
|
||||
}
|
||||
|
||||
/// Add a single output item
|
||||
pub fn add_output(mut self, item: ResponseOutputItem) -> Self {
|
||||
self.output.push(item);
|
||||
self
|
||||
}
|
||||
|
||||
/// Set whether parallel tool calls are enabled
|
||||
pub fn parallel_tool_calls(mut self, enabled: bool) -> Self {
|
||||
self.parallel_tool_calls = enabled;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set previous response ID (if continuation)
|
||||
pub fn previous_response_id(mut self, id: impl Into<String>) -> Self {
|
||||
self.previous_response_id = Some(id.into());
|
||||
self
|
||||
}
|
||||
|
||||
/// Set reasoning information
|
||||
pub fn reasoning(mut self, reasoning: ReasoningInfo) -> Self {
|
||||
self.reasoning = Some(reasoning);
|
||||
self
|
||||
}
|
||||
|
||||
/// Set whether the response is stored
|
||||
pub fn store(mut self, store: bool) -> Self {
|
||||
self.store = store;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set temperature setting
|
||||
pub fn temperature(mut self, temperature: f32) -> Self {
|
||||
self.temperature = Some(temperature);
|
||||
self
|
||||
}
|
||||
|
||||
/// Set text format settings if provided (handles Option)
|
||||
pub fn maybe_text(mut self, text: Option<TextConfig>) -> Self {
|
||||
if let Some(t) = text {
|
||||
self.text = Some(t);
|
||||
}
|
||||
self
|
||||
}
|
||||
|
||||
/// Set tool choice setting
|
||||
pub fn tool_choice(mut self, tool_choice: impl Into<String>) -> Self {
|
||||
self.tool_choice = tool_choice.into();
|
||||
self
|
||||
}
|
||||
|
||||
/// Set available tools
|
||||
pub fn tools(mut self, tools: Vec<ResponseTool>) -> Self {
|
||||
self.tools = tools;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set top-p setting
|
||||
pub fn top_p(mut self, top_p: f32) -> Self {
|
||||
self.top_p = Some(top_p);
|
||||
self
|
||||
}
|
||||
|
||||
/// Set truncation strategy
|
||||
pub fn truncation(mut self, truncation: impl Into<String>) -> Self {
|
||||
self.truncation = Some(truncation.into());
|
||||
self
|
||||
}
|
||||
|
||||
/// Set usage information
|
||||
pub fn usage(mut self, usage: ResponsesUsage) -> Self {
|
||||
self.usage = Some(usage);
|
||||
self
|
||||
}
|
||||
|
||||
/// Set usage if provided (handles Option)
|
||||
pub fn maybe_usage(mut self, usage: Option<ResponsesUsage>) -> Self {
|
||||
if let Some(u) = usage {
|
||||
self.usage = Some(u);
|
||||
}
|
||||
self
|
||||
}
|
||||
|
||||
/// Copy from request if provided (handles Option)
|
||||
pub fn maybe_copy_from_request(mut self, request: Option<&ResponsesRequest>) -> Self {
|
||||
if let Some(req) = request {
|
||||
self = self.copy_from_request(req);
|
||||
}
|
||||
self
|
||||
}
|
||||
|
||||
/// Set user identifier
|
||||
pub fn user(mut self, user: impl Into<String>) -> Self {
|
||||
self.user = Some(user.into());
|
||||
self
|
||||
}
|
||||
|
||||
/// Set safety identifier
|
||||
pub fn safety_identifier(mut self, identifier: impl Into<String>) -> Self {
|
||||
self.safety_identifier = Some(identifier.into());
|
||||
self
|
||||
}
|
||||
|
||||
/// Set metadata
|
||||
pub fn metadata(mut self, metadata: HashMap<String, Value>) -> Self {
|
||||
self.metadata = metadata;
|
||||
self
|
||||
}
|
||||
|
||||
/// Add a single metadata entry
|
||||
pub fn add_metadata(mut self, key: impl Into<String>, value: Value) -> Self {
|
||||
self.metadata.insert(key.into(), value);
|
||||
self
|
||||
}
|
||||
|
||||
/// Build the ResponsesResponse
|
||||
pub fn build(self) -> ResponsesResponse {
|
||||
ResponsesResponse {
|
||||
id: self.id,
|
||||
object: self.object,
|
||||
created_at: self.created_at,
|
||||
status: self.status,
|
||||
error: self.error,
|
||||
incomplete_details: self.incomplete_details,
|
||||
instructions: self.instructions,
|
||||
max_output_tokens: self.max_output_tokens,
|
||||
model: self.model,
|
||||
output: self.output,
|
||||
parallel_tool_calls: self.parallel_tool_calls,
|
||||
previous_response_id: self.previous_response_id,
|
||||
reasoning: self.reasoning,
|
||||
store: self.store,
|
||||
temperature: self.temperature,
|
||||
text: self.text,
|
||||
tool_choice: self.tool_choice,
|
||||
tools: self.tools,
|
||||
top_p: self.top_p,
|
||||
truncation: self.truncation,
|
||||
usage: self.usage,
|
||||
user: self.user,
|
||||
safety_identifier: self.safety_identifier,
|
||||
metadata: self.metadata,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Tests
|
||||
// ============================================================================
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_build_minimal() {
|
||||
let response = ResponsesResponse::builder("resp_123", "gpt-4").build();
|
||||
|
||||
assert_eq!(response.id, "resp_123");
|
||||
assert_eq!(response.model, "gpt-4");
|
||||
assert_eq!(response.object, "response");
|
||||
assert_eq!(response.status, ResponseStatus::InProgress);
|
||||
assert!(response.output.is_empty());
|
||||
assert!(response.parallel_tool_calls);
|
||||
assert!(response.store);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_build_complete() {
|
||||
let response = ResponsesResponse::builder("resp_123", "gpt-4")
|
||||
.status(ResponseStatus::Completed)
|
||||
.instructions("You are a helpful assistant")
|
||||
.max_output_tokens(1000)
|
||||
.temperature(0.7)
|
||||
.top_p(0.9)
|
||||
.parallel_tool_calls(false)
|
||||
.store(false)
|
||||
.build();
|
||||
|
||||
assert_eq!(response.status, ResponseStatus::Completed);
|
||||
assert_eq!(
|
||||
response.instructions.as_ref().unwrap(),
|
||||
"You are a helpful assistant"
|
||||
);
|
||||
assert_eq!(response.max_output_tokens, Some(1000));
|
||||
assert_eq!(response.temperature, Some(0.7));
|
||||
assert_eq!(response.top_p, Some(0.9));
|
||||
assert!(!response.parallel_tool_calls);
|
||||
assert!(!response.store);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_copy_from_request() {
|
||||
let request = ResponsesRequest {
|
||||
model: "gpt-4".to_string(),
|
||||
input: ResponseInput::Text("test".to_string()),
|
||||
instructions: Some("Be helpful".to_string()),
|
||||
max_output_tokens: Some(500),
|
||||
temperature: Some(0.8),
|
||||
top_p: Some(0.95),
|
||||
parallel_tool_calls: Some(false),
|
||||
store: Some(false),
|
||||
user: Some("user_123".to_string()),
|
||||
metadata: Some(HashMap::from([(
|
||||
"key".to_string(),
|
||||
serde_json::json!("value"),
|
||||
)])),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let response = ResponsesResponse::builder("resp_456", "gpt-4")
|
||||
.copy_from_request(&request)
|
||||
.status(ResponseStatus::Completed)
|
||||
.build();
|
||||
|
||||
assert_eq!(response.instructions.as_ref().unwrap(), "Be helpful");
|
||||
assert_eq!(response.max_output_tokens, Some(500));
|
||||
assert_eq!(response.temperature, Some(0.8));
|
||||
assert_eq!(response.top_p, Some(0.95));
|
||||
assert!(!response.parallel_tool_calls);
|
||||
assert!(!response.store);
|
||||
assert_eq!(response.user.as_ref().unwrap(), "user_123");
|
||||
assert_eq!(
|
||||
response.metadata.get("key").unwrap(),
|
||||
&serde_json::json!("value")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_add_output_items() {
|
||||
let response = ResponsesResponse::builder("resp_789", "gpt-4")
|
||||
.add_output(ResponseOutputItem::Message {
|
||||
id: "msg_1".to_string(),
|
||||
role: "assistant".to_string(),
|
||||
content: vec![],
|
||||
status: "completed".to_string(),
|
||||
})
|
||||
.add_output(ResponseOutputItem::Message {
|
||||
id: "msg_2".to_string(),
|
||||
role: "assistant".to_string(),
|
||||
content: vec![],
|
||||
status: "completed".to_string(),
|
||||
})
|
||||
.build();
|
||||
|
||||
assert_eq!(response.output.len(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_add_metadata() {
|
||||
let response = ResponsesResponse::builder("resp_101", "gpt-4")
|
||||
.add_metadata("key1", serde_json::json!("value1"))
|
||||
.add_metadata("key2", serde_json::json!(42))
|
||||
.build();
|
||||
|
||||
assert_eq!(response.metadata.len(), 2);
|
||||
assert_eq!(response.metadata.get("key1").unwrap(), "value1");
|
||||
assert_eq!(response.metadata.get("key2").unwrap(), 42);
|
||||
}
|
||||
}
|
||||
@@ -1,799 +0,0 @@
|
||||
use std::collections::HashMap;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
use validator::Validate;
|
||||
|
||||
use super::{
|
||||
common::{
|
||||
default_model, default_true, validate_stop, ChatLogProbs, ContentPart, Function,
|
||||
FunctionCall, FunctionChoice, GenerationRequest, ResponseFormat, StreamOptions,
|
||||
StringOrArray, Tool, ToolCall, ToolCallDelta, ToolChoice, ToolChoiceValue, ToolReference,
|
||||
Usage,
|
||||
},
|
||||
sampling_params::{validate_top_k_value, validate_top_p_value},
|
||||
};
|
||||
use crate::protocols::{
|
||||
builders::{ChatCompletionResponseBuilder, ChatCompletionStreamResponseBuilder},
|
||||
validated::Normalizable,
|
||||
};
|
||||
|
||||
// ============================================================================
|
||||
// Chat Messages
|
||||
// ============================================================================
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
#[serde(tag = "role")]
|
||||
pub enum ChatMessage {
|
||||
#[serde(rename = "system")]
|
||||
System {
|
||||
content: MessageContent,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
name: Option<String>,
|
||||
},
|
||||
#[serde(rename = "user")]
|
||||
User {
|
||||
content: MessageContent,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
name: Option<String>,
|
||||
},
|
||||
#[serde(rename = "assistant")]
|
||||
Assistant {
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
content: Option<MessageContent>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
name: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
tool_calls: Option<Vec<ToolCall>>,
|
||||
/// Reasoning content for O1-style models (SGLang extension)
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
reasoning_content: Option<String>,
|
||||
},
|
||||
#[serde(rename = "tool")]
|
||||
Tool {
|
||||
content: MessageContent,
|
||||
tool_call_id: String,
|
||||
},
|
||||
#[serde(rename = "function")]
|
||||
Function { content: String, name: String },
|
||||
#[serde(rename = "developer")]
|
||||
Developer {
|
||||
content: MessageContent,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
tools: Option<Vec<Tool>>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
name: Option<String>,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize, PartialEq)]
|
||||
#[serde(untagged)]
|
||||
pub enum MessageContent {
|
||||
Text(String),
|
||||
Parts(Vec<ContentPart>),
|
||||
}
|
||||
|
||||
impl MessageContent {
|
||||
/// Returns the text content, cloning only when necessary.
|
||||
/// For simple text, returns a clone of the string.
|
||||
/// For parts, concatenates text parts with spaces.
|
||||
/// Optimized to avoid intermediate Vec allocation.
|
||||
pub fn to_simple_string(&self) -> String {
|
||||
match self {
|
||||
MessageContent::Text(text) => text.clone(),
|
||||
MessageContent::Parts(parts) => {
|
||||
// Use fold to build string directly without intermediate Vec allocation
|
||||
let mut result = String::new();
|
||||
let mut first = true;
|
||||
for part in parts {
|
||||
if let ContentPart::Text { text } = part {
|
||||
if !first {
|
||||
result.push(' ');
|
||||
}
|
||||
result.push_str(text);
|
||||
first = false;
|
||||
}
|
||||
}
|
||||
result
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Appends text content directly to a buffer, avoiding intermediate allocations.
|
||||
/// Returns true if any content was appended.
|
||||
#[inline]
|
||||
pub fn append_text_to(&self, buffer: &mut String) -> bool {
|
||||
match self {
|
||||
MessageContent::Text(text) => {
|
||||
if !text.is_empty() {
|
||||
buffer.push_str(text);
|
||||
true
|
||||
} else {
|
||||
false
|
||||
}
|
||||
}
|
||||
MessageContent::Parts(parts) => {
|
||||
let mut appended = false;
|
||||
for part in parts {
|
||||
if let ContentPart::Text { text } = part {
|
||||
if !text.is_empty() {
|
||||
if appended {
|
||||
buffer.push(' ');
|
||||
}
|
||||
buffer.push_str(text);
|
||||
appended = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
appended
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns true if this content contains any non-empty text.
|
||||
#[inline]
|
||||
pub fn has_text(&self) -> bool {
|
||||
match self {
|
||||
MessageContent::Text(text) => !text.is_empty(),
|
||||
MessageContent::Parts(parts) => parts
|
||||
.iter()
|
||||
.any(|part| matches!(part, ContentPart::Text { text } if !text.is_empty())),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Chat Completion Request
|
||||
// ============================================================================
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize, Default, Validate)]
|
||||
#[validate(schema(function = "validate_chat_cross_parameters"))]
|
||||
pub struct ChatCompletionRequest {
|
||||
/// A list of messages comprising the conversation so far
|
||||
#[validate(custom(function = "validate_messages"))]
|
||||
pub messages: Vec<ChatMessage>,
|
||||
|
||||
/// ID of the model to use
|
||||
#[serde(default = "default_model")]
|
||||
pub model: String,
|
||||
|
||||
/// Number between -2.0 and 2.0. Positive values penalize new tokens based on their existing frequency in the text so far
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
#[validate(range(min = -2.0, max = 2.0))]
|
||||
pub frequency_penalty: Option<f32>,
|
||||
|
||||
/// Deprecated: Replaced by tool_choice
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
#[deprecated(note = "Use tool_choice instead")]
|
||||
pub function_call: Option<FunctionCall>,
|
||||
|
||||
/// Deprecated: Replaced by tools
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
#[deprecated(note = "Use tools instead")]
|
||||
pub functions: Option<Vec<Function>>,
|
||||
|
||||
/// Modify the likelihood of specified tokens appearing in the completion
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub logit_bias: Option<HashMap<String, f32>>,
|
||||
|
||||
/// Whether to return log probabilities of the output tokens
|
||||
#[serde(default)]
|
||||
pub logprobs: bool,
|
||||
|
||||
/// Deprecated: Replaced by max_completion_tokens
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
#[deprecated(note = "Use max_completion_tokens instead")]
|
||||
#[validate(range(min = 1))]
|
||||
pub max_tokens: Option<u32>,
|
||||
|
||||
/// An upper bound for the number of tokens that can be generated for a completion
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
#[validate(range(min = 1))]
|
||||
pub max_completion_tokens: Option<u32>,
|
||||
|
||||
/// Developer-defined tags and values used for filtering completions in the dashboard
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub metadata: Option<HashMap<String, String>>,
|
||||
|
||||
/// Output types that you would like the model to generate for this request
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub modalities: Option<Vec<String>>,
|
||||
|
||||
/// How many chat completion choices to generate for each input message
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
#[validate(range(min = 1, max = 10))]
|
||||
pub n: Option<u32>,
|
||||
|
||||
/// Whether to enable parallel function calling during tool use
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub parallel_tool_calls: Option<bool>,
|
||||
|
||||
/// Number between -2.0 and 2.0. Positive values penalize new tokens based on whether they appear in the text so far
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
#[validate(range(min = -2.0, max = 2.0))]
|
||||
pub presence_penalty: Option<f32>,
|
||||
|
||||
/// Cache key for prompts (beta feature)
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub prompt_cache_key: Option<String>,
|
||||
|
||||
/// Effort level for reasoning models (low, medium, high)
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub reasoning_effort: Option<String>,
|
||||
|
||||
/// An object specifying the format that the model must output
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub response_format: Option<ResponseFormat>,
|
||||
|
||||
/// Safety identifier for content moderation
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub safety_identifier: Option<String>,
|
||||
|
||||
/// Deprecated: This feature is in Legacy mode
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
#[deprecated(note = "This feature is in Legacy mode")]
|
||||
pub seed: Option<i64>,
|
||||
|
||||
/// The service tier to use for this request
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub service_tier: Option<String>,
|
||||
|
||||
/// Up to 4 sequences where the API will stop generating further tokens
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
#[validate(custom(function = "validate_stop"))]
|
||||
pub stop: Option<StringOrArray>,
|
||||
|
||||
/// If set, partial message deltas will be sent
|
||||
#[serde(default)]
|
||||
pub stream: bool,
|
||||
|
||||
/// Options for streaming response
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub stream_options: Option<StreamOptions>,
|
||||
|
||||
/// What sampling temperature to use, between 0 and 2
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
#[validate(range(min = 0.0, max = 2.0))]
|
||||
pub temperature: Option<f32>,
|
||||
|
||||
/// Controls which (if any) tool is called by the model
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub tool_choice: Option<ToolChoice>,
|
||||
|
||||
/// A list of tools the model may call
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub tools: Option<Vec<Tool>>,
|
||||
|
||||
/// An integer between 0 and 20 specifying the number of most likely tokens to return
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
#[validate(range(min = 0, max = 20))]
|
||||
pub top_logprobs: Option<u32>,
|
||||
|
||||
/// An alternative to sampling with temperature
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
#[validate(custom(function = "validate_top_p_value"))]
|
||||
pub top_p: Option<f32>,
|
||||
|
||||
/// Verbosity level for debugging
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub verbosity: Option<i32>,
|
||||
|
||||
// =============================================================================
|
||||
// Engine-Specific Sampling Parameters
|
||||
// =============================================================================
|
||||
// These parameters are extensions beyond the OpenAI API specification and
|
||||
// control model generation behavior in engine-specific ways.
|
||||
// =============================================================================
|
||||
/// Top-k sampling parameter (-1 to disable)
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
#[validate(custom(function = "validate_top_k_value"))]
|
||||
pub top_k: Option<i32>,
|
||||
|
||||
/// Min-p nucleus sampling parameter
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
#[validate(range(min = 0.0, max = 1.0))]
|
||||
pub min_p: Option<f32>,
|
||||
|
||||
/// Minimum number of tokens to generate
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
#[validate(range(min = 1))]
|
||||
pub min_tokens: Option<u32>,
|
||||
|
||||
/// Repetition penalty for reducing repetitive text
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
#[validate(range(min = 0.0, max = 2.0))]
|
||||
pub repetition_penalty: Option<f32>,
|
||||
|
||||
/// Regex constraint for output generation
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub regex: Option<String>,
|
||||
|
||||
/// EBNF grammar constraint for structured output
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub ebnf: Option<String>,
|
||||
|
||||
/// Specific token IDs to use as stop conditions
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub stop_token_ids: Option<Vec<u32>>,
|
||||
|
||||
/// Skip trimming stop tokens from output
|
||||
#[serde(default)]
|
||||
pub no_stop_trim: bool,
|
||||
|
||||
/// Ignore end-of-sequence tokens during generation
|
||||
#[serde(default)]
|
||||
pub ignore_eos: bool,
|
||||
|
||||
/// Continue generating from final assistant message
|
||||
#[serde(default)]
|
||||
pub continue_final_message: bool,
|
||||
|
||||
/// Skip special tokens during detokenization
|
||||
#[serde(default = "default_true")]
|
||||
pub skip_special_tokens: bool,
|
||||
|
||||
/// Path to LoRA adapter(s) for model customization
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub lora_path: Option<String>,
|
||||
|
||||
/// Session parameters for continual prompting
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub session_params: Option<HashMap<String, Value>>,
|
||||
|
||||
/// Separate reasoning content from final answer (O1-style models)
|
||||
#[serde(default = "default_true")]
|
||||
pub separate_reasoning: bool,
|
||||
|
||||
/// Stream reasoning tokens during generation
|
||||
#[serde(default = "default_true")]
|
||||
pub stream_reasoning: bool,
|
||||
|
||||
/// Chat template kwargs
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub chat_template_kwargs: Option<HashMap<String, Value>>,
|
||||
|
||||
/// Return model hidden states
|
||||
#[serde(default)]
|
||||
pub return_hidden_states: bool,
|
||||
|
||||
/// Random seed for sampling for deterministic outputs
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub sampling_seed: Option<u64>,
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Validation Functions
|
||||
// ============================================================================
|
||||
|
||||
/// Validates messages array is not empty and has valid content
|
||||
fn validate_messages(messages: &[ChatMessage]) -> Result<(), validator::ValidationError> {
|
||||
if messages.is_empty() {
|
||||
return Err(validator::ValidationError::new("messages cannot be empty"));
|
||||
}
|
||||
|
||||
for msg in messages.iter() {
|
||||
if let ChatMessage::User { content, .. } = msg {
|
||||
match content {
|
||||
MessageContent::Text(text) if text.is_empty() => {
|
||||
return Err(validator::ValidationError::new(
|
||||
"message content cannot be empty",
|
||||
));
|
||||
}
|
||||
MessageContent::Parts(parts) if parts.is_empty() => {
|
||||
return Err(validator::ValidationError::new(
|
||||
"message content parts cannot be empty",
|
||||
));
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Schema-level validation for cross-field dependencies
|
||||
fn validate_chat_cross_parameters(
|
||||
req: &ChatCompletionRequest,
|
||||
) -> Result<(), validator::ValidationError> {
|
||||
// 1. Validate logprobs dependency
|
||||
if req.top_logprobs.is_some() && !req.logprobs {
|
||||
let mut e = validator::ValidationError::new("top_logprobs_requires_logprobs");
|
||||
e.message = Some("top_logprobs is only allowed when logprobs is enabled".into());
|
||||
return Err(e);
|
||||
}
|
||||
|
||||
// 2. Validate stream_options dependency
|
||||
if req.stream_options.is_some() && !req.stream {
|
||||
let mut e = validator::ValidationError::new("stream_options_requires_stream");
|
||||
e.message =
|
||||
Some("The 'stream_options' parameter is only allowed when 'stream' is enabled".into());
|
||||
return Err(e);
|
||||
}
|
||||
|
||||
// 3. Validate token limits - min <= max
|
||||
if let (Some(min), Some(max)) = (req.min_tokens, req.max_completion_tokens) {
|
||||
if min > max {
|
||||
let mut e = validator::ValidationError::new("min_tokens_exceeds_max");
|
||||
e.message = Some("min_tokens cannot exceed max_tokens/max_completion_tokens".into());
|
||||
return Err(e);
|
||||
}
|
||||
}
|
||||
|
||||
// 4. Validate structured output conflicts
|
||||
let has_json_format = matches!(
|
||||
req.response_format,
|
||||
Some(ResponseFormat::JsonObject | ResponseFormat::JsonSchema { .. })
|
||||
);
|
||||
|
||||
if has_json_format && req.regex.is_some() {
|
||||
let mut e = validator::ValidationError::new("regex_conflicts_with_json");
|
||||
e.message = Some("cannot use regex constraint with JSON response format".into());
|
||||
return Err(e);
|
||||
}
|
||||
|
||||
if has_json_format && req.ebnf.is_some() {
|
||||
let mut e = validator::ValidationError::new("ebnf_conflicts_with_json");
|
||||
e.message = Some("cannot use EBNF constraint with JSON response format".into());
|
||||
return Err(e);
|
||||
}
|
||||
|
||||
// 5. Validate mutually exclusive structured output constraints
|
||||
let constraint_count = [
|
||||
req.regex.is_some(),
|
||||
req.ebnf.is_some(),
|
||||
matches!(req.response_format, Some(ResponseFormat::JsonSchema { .. })),
|
||||
]
|
||||
.iter()
|
||||
.filter(|&&x| x)
|
||||
.count();
|
||||
|
||||
if constraint_count > 1 {
|
||||
let mut e = validator::ValidationError::new("multiple_constraints");
|
||||
e.message = Some("only one structured output constraint (regex, ebnf, or json_schema) can be active at a time".into());
|
||||
return Err(e);
|
||||
}
|
||||
|
||||
// 6. Validate response format JSON schema name
|
||||
if let Some(ResponseFormat::JsonSchema { json_schema }) = &req.response_format {
|
||||
if json_schema.name.is_empty() {
|
||||
let mut e = validator::ValidationError::new("json_schema_name_empty");
|
||||
e.message = Some("JSON schema name cannot be empty".into());
|
||||
return Err(e);
|
||||
}
|
||||
}
|
||||
|
||||
// 7. Validate tool_choice requires tools (except for "none")
|
||||
if let Some(ref tool_choice) = req.tool_choice {
|
||||
let has_tools = req.tools.as_ref().is_some_and(|t| !t.is_empty());
|
||||
|
||||
// Check if tool_choice is anything other than "none"
|
||||
let is_some_choice = !matches!(tool_choice, ToolChoice::Value(ToolChoiceValue::None));
|
||||
|
||||
if is_some_choice && !has_tools {
|
||||
let mut e = validator::ValidationError::new("tool_choice_requires_tools");
|
||||
e.message = Some("Invalid value for 'tool_choice': 'tool_choice' is only allowed when 'tools' are specified.".into());
|
||||
return Err(e);
|
||||
}
|
||||
|
||||
// Additional validation when tools are present
|
||||
if has_tools {
|
||||
let tools = req.tools.as_ref().unwrap();
|
||||
|
||||
match tool_choice {
|
||||
ToolChoice::Function { function, .. } => {
|
||||
// Validate that the specified function name exists in tools
|
||||
let function_exists = tools.iter().any(|tool| {
|
||||
tool.tool_type == "function" && tool.function.name == function.name
|
||||
});
|
||||
|
||||
if !function_exists {
|
||||
let mut e =
|
||||
validator::ValidationError::new("tool_choice_function_not_found");
|
||||
e.message = Some(
|
||||
format!(
|
||||
"Invalid value for 'tool_choice': function '{}' not found in 'tools'.",
|
||||
function.name
|
||||
)
|
||||
.into(),
|
||||
);
|
||||
return Err(e);
|
||||
}
|
||||
}
|
||||
ToolChoice::AllowedTools {
|
||||
mode,
|
||||
tools: allowed_tools,
|
||||
..
|
||||
} => {
|
||||
// Validate mode is "auto" or "required"
|
||||
if mode != "auto" && mode != "required" {
|
||||
let mut e = validator::ValidationError::new("tool_choice_invalid_mode");
|
||||
e.message = Some(format!(
|
||||
"Invalid value for 'tool_choice.mode': must be 'auto' or 'required', got '{}'.",
|
||||
mode
|
||||
).into());
|
||||
return Err(e);
|
||||
}
|
||||
|
||||
// Validate that all ToolReferences are Function type (Chat API only supports function tools)
|
||||
for tool_ref in allowed_tools {
|
||||
match tool_ref {
|
||||
ToolReference::Function { name } => {
|
||||
// Validate that the function exists in tools array
|
||||
let tool_exists = tools.iter().any(|tool| {
|
||||
tool.tool_type == "function" && tool.function.name == *name
|
||||
});
|
||||
|
||||
if !tool_exists {
|
||||
let mut e = validator::ValidationError::new(
|
||||
"tool_choice_tool_not_found",
|
||||
);
|
||||
e.message = Some(
|
||||
format!(
|
||||
"Invalid value for 'tool_choice.tools': tool '{}' not found in 'tools'.",
|
||||
name
|
||||
)
|
||||
.into(),
|
||||
);
|
||||
return Err(e);
|
||||
}
|
||||
}
|
||||
_ => {
|
||||
// Chat Completion API only supports function tools in tool_choice
|
||||
let mut e = validator::ValidationError::new(
|
||||
"tool_choice_invalid_tool_type",
|
||||
);
|
||||
e.message = Some(
|
||||
format!(
|
||||
"Invalid value for 'tool_choice.tools': Chat Completion API only supports function tools, got '{}'.",
|
||||
tool_ref.identifier()
|
||||
)
|
||||
.into(),
|
||||
);
|
||||
return Err(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Normalizable Implementation
|
||||
// ============================================================================
|
||||
|
||||
impl Normalizable for ChatCompletionRequest {
|
||||
/// Normalize the request by applying migrations and defaults:
|
||||
/// 1. Migrate deprecated fields to their replacements
|
||||
/// 2. Clear deprecated fields and log warnings
|
||||
/// 3. Apply OpenAI defaults for tool_choice
|
||||
fn normalize(&mut self) {
|
||||
// Migrate deprecated max_tokens → max_completion_tokens
|
||||
#[allow(deprecated)]
|
||||
if self.max_completion_tokens.is_none() && self.max_tokens.is_some() {
|
||||
self.max_completion_tokens = self.max_tokens;
|
||||
self.max_tokens = None; // Clear deprecated field
|
||||
}
|
||||
|
||||
// Migrate deprecated functions → tools
|
||||
#[allow(deprecated)]
|
||||
if self.tools.is_none() && self.functions.is_some() {
|
||||
tracing::warn!("functions is deprecated, use tools instead");
|
||||
self.tools = self.functions.as_ref().map(|functions| {
|
||||
functions
|
||||
.iter()
|
||||
.map(|func| Tool {
|
||||
tool_type: "function".to_string(),
|
||||
function: func.clone(),
|
||||
})
|
||||
.collect()
|
||||
});
|
||||
self.functions = None; // Clear deprecated field
|
||||
}
|
||||
|
||||
// Migrate deprecated function_call → tool_choice
|
||||
#[allow(deprecated)]
|
||||
if self.tool_choice.is_none() && self.function_call.is_some() {
|
||||
tracing::warn!("function_call is deprecated, use tool_choice instead");
|
||||
self.tool_choice = self.function_call.as_ref().map(|fc| match fc {
|
||||
FunctionCall::None => ToolChoice::Value(ToolChoiceValue::None),
|
||||
FunctionCall::Auto => ToolChoice::Value(ToolChoiceValue::Auto),
|
||||
FunctionCall::Function { name } => ToolChoice::Function {
|
||||
tool_type: "function".to_string(),
|
||||
function: FunctionChoice { name: name.clone() },
|
||||
},
|
||||
});
|
||||
self.function_call = None; // Clear deprecated field
|
||||
}
|
||||
|
||||
// Apply tool_choice defaults
|
||||
if self.tool_choice.is_none() {
|
||||
if let Some(tools) = &self.tools {
|
||||
let choice_value = if !tools.is_empty() {
|
||||
ToolChoiceValue::Auto
|
||||
} else {
|
||||
ToolChoiceValue::None
|
||||
};
|
||||
self.tool_choice = Some(ToolChoice::Value(choice_value));
|
||||
}
|
||||
// If tools is None, leave tool_choice as None (don't set it)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// GenerationRequest Trait Implementation
|
||||
// ============================================================================
|
||||
|
||||
impl GenerationRequest for ChatCompletionRequest {
|
||||
fn is_stream(&self) -> bool {
|
||||
self.stream
|
||||
}
|
||||
|
||||
fn get_model(&self) -> Option<&str> {
|
||||
Some(&self.model)
|
||||
}
|
||||
|
||||
fn extract_text_for_routing(&self) -> String {
|
||||
// Extract text from messages for routing decisions
|
||||
// Use a single buffer to avoid intermediate Vec<String> allocations
|
||||
let mut buffer = String::new();
|
||||
let mut has_content = false;
|
||||
|
||||
for msg in &self.messages {
|
||||
match msg {
|
||||
ChatMessage::System { content, .. }
|
||||
| ChatMessage::User { content, .. }
|
||||
| ChatMessage::Tool { content, .. }
|
||||
| ChatMessage::Developer { content, .. } => {
|
||||
if has_content && content.has_text() {
|
||||
buffer.push(' ');
|
||||
}
|
||||
if content.append_text_to(&mut buffer) {
|
||||
has_content = true;
|
||||
}
|
||||
}
|
||||
ChatMessage::Assistant {
|
||||
content,
|
||||
reasoning_content,
|
||||
..
|
||||
} => {
|
||||
// Append main content
|
||||
if let Some(c) = content {
|
||||
if has_content && c.has_text() {
|
||||
buffer.push(' ');
|
||||
}
|
||||
if c.append_text_to(&mut buffer) {
|
||||
has_content = true;
|
||||
}
|
||||
}
|
||||
// Append reasoning content
|
||||
if let Some(reasoning) = reasoning_content {
|
||||
if !reasoning.is_empty() {
|
||||
if has_content {
|
||||
buffer.push(' ');
|
||||
}
|
||||
buffer.push_str(reasoning);
|
||||
has_content = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
ChatMessage::Function { content, .. } => {
|
||||
if !content.is_empty() {
|
||||
if has_content {
|
||||
buffer.push(' ');
|
||||
}
|
||||
buffer.push_str(content);
|
||||
has_content = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
buffer
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Response Types
|
||||
// ============================================================================
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct ChatCompletionResponse {
|
||||
pub id: String,
|
||||
pub object: String, // "chat.completion"
|
||||
pub created: u64,
|
||||
pub model: String,
|
||||
pub choices: Vec<ChatChoice>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub usage: Option<Usage>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub system_fingerprint: Option<String>,
|
||||
}
|
||||
|
||||
impl ChatCompletionResponse {
|
||||
/// Create a new builder for ChatCompletionResponse
|
||||
pub fn builder(
|
||||
id: impl Into<String>,
|
||||
model: impl Into<String>,
|
||||
) -> ChatCompletionResponseBuilder {
|
||||
ChatCompletionResponseBuilder::new(id, model)
|
||||
}
|
||||
}
|
||||
|
||||
/// Response message structure for ChatCompletionResponse (different from request ChatMessage)
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct ChatCompletionMessage {
|
||||
pub role: String, // Always "assistant" for responses
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub content: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub tool_calls: Option<Vec<ToolCall>>,
|
||||
pub reasoning_content: Option<String>,
|
||||
// Note: function_call is deprecated and not included
|
||||
// Note: refusal, annotations, audio are not added yet
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct ChatChoice {
|
||||
pub index: u32,
|
||||
pub message: ChatCompletionMessage,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub logprobs: Option<ChatLogProbs>,
|
||||
pub finish_reason: Option<String>, // "stop", "length", "tool_calls", "content_filter", "function_call"
|
||||
/// Information about which stop condition was matched
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub matched_stop: Option<Value>, // Can be string or integer
|
||||
/// Hidden states from the model (SGLang extension)
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub hidden_states: Option<Vec<f32>>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct ChatCompletionStreamResponse {
|
||||
pub id: String,
|
||||
pub object: String, // "chat.completion.chunk"
|
||||
pub created: u64,
|
||||
pub model: String,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub system_fingerprint: Option<String>,
|
||||
pub choices: Vec<ChatStreamChoice>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub usage: Option<Usage>,
|
||||
}
|
||||
|
||||
impl ChatCompletionStreamResponse {
|
||||
/// Create a new builder for ChatCompletionStreamResponse
|
||||
pub fn builder(
|
||||
id: impl Into<String>,
|
||||
model: impl Into<String>,
|
||||
) -> ChatCompletionStreamResponseBuilder {
|
||||
ChatCompletionStreamResponseBuilder::new(id, model)
|
||||
}
|
||||
}
|
||||
|
||||
/// Delta structure for streaming chat completion responses
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct ChatMessageDelta {
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub role: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub content: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub tool_calls: Option<Vec<ToolCallDelta>>,
|
||||
pub reasoning_content: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct ChatStreamChoice {
|
||||
pub index: u32,
|
||||
pub delta: ChatMessageDelta,
|
||||
pub logprobs: Option<ChatLogProbs>,
|
||||
pub finish_reason: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub matched_stop: Option<Value>,
|
||||
}
|
||||
@@ -1,121 +0,0 @@
|
||||
//! Classify API protocol definitions.
|
||||
//!
|
||||
//! This module defines the request and response types for the `/v1/classify` API,
|
||||
//! which is compatible with vLLM's classification endpoint.
|
||||
//!
|
||||
//! Classification reuses the embedding backend - the scheduler returns logits as
|
||||
//! "embeddings", and the classify layer applies softmax + label mapping.
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
|
||||
use super::common::{GenerationRequest, UsageInfo};
|
||||
|
||||
// ============================================================================
|
||||
// Classify API
|
||||
// ============================================================================
|
||||
|
||||
/// Classification request - compatible with vLLM's /v1/classify API
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct ClassifyRequest {
|
||||
/// ID of the model to use
|
||||
pub model: String,
|
||||
|
||||
/// Input can be a string, array of strings, or token IDs
|
||||
/// - Single string: "text to classify"
|
||||
/// - Array of strings: ["text1", "text2"]
|
||||
/// - Token IDs: [1, 2, 3] (advanced usage)
|
||||
pub input: Value,
|
||||
|
||||
/// Optional user identifier
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub user: Option<String>,
|
||||
|
||||
/// SGLang extension: request id for tracking
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub rid: Option<String>,
|
||||
|
||||
/// SGLang extension: request priority
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub priority: Option<i32>,
|
||||
|
||||
/// SGLang extension: enable/disable logging of metrics
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub log_metrics: Option<bool>,
|
||||
}
|
||||
|
||||
impl GenerationRequest for ClassifyRequest {
|
||||
fn is_stream(&self) -> bool {
|
||||
false // Classification is always non-streaming
|
||||
}
|
||||
|
||||
fn get_model(&self) -> Option<&str> {
|
||||
Some(&self.model)
|
||||
}
|
||||
|
||||
fn extract_text_for_routing(&self) -> String {
|
||||
match &self.input {
|
||||
Value::String(s) => s.clone(),
|
||||
Value::Array(arr) => arr
|
||||
.iter()
|
||||
.filter_map(|v| v.as_str())
|
||||
.collect::<Vec<_>>()
|
||||
.join(" "),
|
||||
_ => String::new(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Classify Response
|
||||
// ============================================================================
|
||||
|
||||
/// Single classification result
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ClassifyData {
|
||||
/// Index of this result (for batch requests)
|
||||
pub index: u32,
|
||||
/// Predicted class label (from id2label mapping)
|
||||
pub label: String,
|
||||
/// Probability distribution over all classes (softmax of logits)
|
||||
pub probs: Vec<f32>,
|
||||
/// Number of classes
|
||||
pub num_classes: u32,
|
||||
}
|
||||
|
||||
/// Classification response
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ClassifyResponse {
|
||||
/// Unique request ID (format: "classify-{uuid}")
|
||||
pub id: String,
|
||||
/// Always "list"
|
||||
pub object: String,
|
||||
/// Unix timestamp (seconds since epoch)
|
||||
pub created: u64,
|
||||
/// Model name
|
||||
pub model: String,
|
||||
/// Classification results (one per input in batch)
|
||||
pub data: Vec<ClassifyData>,
|
||||
/// Token usage info
|
||||
pub usage: UsageInfo,
|
||||
}
|
||||
|
||||
impl ClassifyResponse {
|
||||
/// Create a new ClassifyResponse with the given data
|
||||
pub fn new(
|
||||
id: String,
|
||||
model: String,
|
||||
created: u64,
|
||||
data: Vec<ClassifyData>,
|
||||
usage: UsageInfo,
|
||||
) -> Self {
|
||||
Self {
|
||||
id,
|
||||
object: "list".to_string(),
|
||||
created,
|
||||
model,
|
||||
data,
|
||||
usage,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,524 +0,0 @@
|
||||
use std::collections::HashMap;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
use validator;
|
||||
|
||||
use super::UNKNOWN_MODEL_ID;
|
||||
|
||||
// ============================================================================
|
||||
// Default value helpers
|
||||
// ============================================================================
|
||||
|
||||
/// Default model value when not specified
|
||||
pub(crate) fn default_model() -> String {
|
||||
UNKNOWN_MODEL_ID.to_string()
|
||||
}
|
||||
|
||||
/// Helper function for serde default value (returns true)
|
||||
pub fn default_true() -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// GenerationRequest Trait
|
||||
// ============================================================================
|
||||
|
||||
/// Trait for unified access to generation request properties
|
||||
/// Implemented by ChatCompletionRequest, CompletionRequest, GenerateRequest,
|
||||
/// EmbeddingRequest, RerankRequest, and ResponsesRequest
|
||||
pub trait GenerationRequest: Send + Sync {
|
||||
/// Check if the request is for streaming
|
||||
fn is_stream(&self) -> bool;
|
||||
|
||||
/// Get the model name if specified
|
||||
fn get_model(&self) -> Option<&str>;
|
||||
|
||||
/// Extract text content for routing decisions
|
||||
fn extract_text_for_routing(&self) -> String;
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// String/Array Utilities
|
||||
// ============================================================================
|
||||
|
||||
/// A type that can be either a single string or an array of strings
|
||||
#[derive(Debug, Clone, PartialEq, Deserialize, Serialize)]
|
||||
#[serde(untagged)]
|
||||
pub enum StringOrArray {
|
||||
String(String),
|
||||
Array(Vec<String>),
|
||||
}
|
||||
|
||||
impl StringOrArray {
|
||||
/// Get the number of items in the StringOrArray
|
||||
pub fn len(&self) -> usize {
|
||||
match self {
|
||||
StringOrArray::String(_) => 1,
|
||||
StringOrArray::Array(arr) => arr.len(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Check if the StringOrArray is empty
|
||||
pub fn is_empty(&self) -> bool {
|
||||
match self {
|
||||
StringOrArray::String(s) => s.is_empty(),
|
||||
StringOrArray::Array(arr) => arr.is_empty(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Convert to a vector of strings (clones the data)
|
||||
pub fn to_vec(&self) -> Vec<String> {
|
||||
match self {
|
||||
StringOrArray::String(s) => vec![s.clone()],
|
||||
StringOrArray::Array(arr) => arr.clone(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns an iterator over string references without cloning.
|
||||
/// Use this instead of `to_vec()` when you only need to iterate.
|
||||
pub fn iter(&self) -> StringOrArrayIter<'_> {
|
||||
StringOrArrayIter {
|
||||
inner: self,
|
||||
index: 0,
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the first string, or None if empty
|
||||
pub fn first(&self) -> Option<&str> {
|
||||
match self {
|
||||
StringOrArray::String(s) => {
|
||||
if s.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(s)
|
||||
}
|
||||
}
|
||||
StringOrArray::Array(arr) => arr.first().map(|s| s.as_str()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Iterator over StringOrArray that yields string references without cloning
|
||||
pub struct StringOrArrayIter<'a> {
|
||||
inner: &'a StringOrArray,
|
||||
index: usize,
|
||||
}
|
||||
|
||||
impl<'a> Iterator for StringOrArrayIter<'a> {
|
||||
type Item = &'a str;
|
||||
|
||||
fn next(&mut self) -> Option<Self::Item> {
|
||||
match self.inner {
|
||||
StringOrArray::String(s) => {
|
||||
if self.index == 0 {
|
||||
self.index = 1;
|
||||
Some(s.as_str())
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
StringOrArray::Array(arr) => {
|
||||
if self.index < arr.len() {
|
||||
let item = &arr[self.index];
|
||||
self.index += 1;
|
||||
Some(item.as_str())
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn size_hint(&self) -> (usize, Option<usize>) {
|
||||
let remaining = match self.inner {
|
||||
StringOrArray::String(_) => 1 - self.index,
|
||||
StringOrArray::Array(arr) => arr.len() - self.index,
|
||||
};
|
||||
(remaining, Some(remaining))
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a> ExactSizeIterator for StringOrArrayIter<'a> {}
|
||||
|
||||
/// Validates stop sequences (max 4, non-empty strings)
|
||||
/// Used by both ChatCompletionRequest and ResponsesRequest
|
||||
pub fn validate_stop(stop: &StringOrArray) -> Result<(), validator::ValidationError> {
|
||||
match stop {
|
||||
StringOrArray::String(s) => {
|
||||
if s.is_empty() {
|
||||
return Err(validator::ValidationError::new(
|
||||
"stop sequences cannot be empty",
|
||||
));
|
||||
}
|
||||
}
|
||||
StringOrArray::Array(arr) => {
|
||||
if arr.len() > 4 {
|
||||
return Err(validator::ValidationError::new(
|
||||
"maximum 4 stop sequences allowed",
|
||||
));
|
||||
}
|
||||
for s in arr {
|
||||
if s.is_empty() {
|
||||
return Err(validator::ValidationError::new(
|
||||
"stop sequences cannot be empty",
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Content Parts (for multimodal messages)
|
||||
// ============================================================================
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize, PartialEq)]
|
||||
#[serde(tag = "type")]
|
||||
pub enum ContentPart {
|
||||
#[serde(rename = "text")]
|
||||
Text { text: String },
|
||||
#[serde(rename = "image_url")]
|
||||
ImageUrl { image_url: ImageUrl },
|
||||
#[serde(rename = "video_url")]
|
||||
VideoUrl { video_url: VideoUrl },
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize, PartialEq)]
|
||||
pub struct ImageUrl {
|
||||
pub url: String,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub detail: Option<String>, // "auto", "low", or "high"
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize, PartialEq)]
|
||||
pub struct VideoUrl {
|
||||
pub url: String,
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Response Format (for structured outputs)
|
||||
// ============================================================================
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
#[serde(tag = "type")]
|
||||
pub enum ResponseFormat {
|
||||
#[serde(rename = "text")]
|
||||
Text,
|
||||
#[serde(rename = "json_object")]
|
||||
JsonObject,
|
||||
#[serde(rename = "json_schema")]
|
||||
JsonSchema { json_schema: JsonSchemaFormat },
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct JsonSchemaFormat {
|
||||
pub name: String,
|
||||
pub schema: Value,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub strict: Option<bool>,
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Streaming
|
||||
// ============================================================================
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct StreamOptions {
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub include_usage: Option<bool>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct ToolCallDelta {
|
||||
pub index: u32,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub id: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
#[serde(rename = "type")]
|
||||
pub tool_type: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub function: Option<FunctionCallDelta>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct FunctionCallDelta {
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub name: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub arguments: Option<String>,
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Tools and Function Calling
|
||||
// ============================================================================
|
||||
|
||||
/// Tool choice value for simple string options
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum ToolChoiceValue {
|
||||
Auto,
|
||||
Required,
|
||||
None,
|
||||
}
|
||||
|
||||
/// Tool choice for both Chat Completion and Responses APIs
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
#[serde(untagged)]
|
||||
pub enum ToolChoice {
|
||||
Value(ToolChoiceValue),
|
||||
Function {
|
||||
#[serde(rename = "type")]
|
||||
tool_type: String, // "function"
|
||||
function: FunctionChoice,
|
||||
},
|
||||
AllowedTools {
|
||||
#[serde(rename = "type")]
|
||||
tool_type: String, // "allowed_tools"
|
||||
mode: String, // "auto" | "required" TODO: need validation
|
||||
tools: Vec<ToolReference>,
|
||||
},
|
||||
}
|
||||
|
||||
impl Default for ToolChoice {
|
||||
fn default() -> Self {
|
||||
Self::Value(ToolChoiceValue::Auto)
|
||||
}
|
||||
}
|
||||
|
||||
impl ToolChoice {
|
||||
/// Serialize tool_choice to string for ResponsesResponse
|
||||
///
|
||||
/// Returns the JSON-serialized tool_choice or "auto" as default
|
||||
pub fn serialize_to_string(tool_choice: &Option<ToolChoice>) -> String {
|
||||
tool_choice
|
||||
.as_ref()
|
||||
.map(|tc| serde_json::to_string(tc).unwrap_or_else(|_| "auto".to_string()))
|
||||
.unwrap_or_else(|| "auto".to_string())
|
||||
}
|
||||
}
|
||||
|
||||
/// Function choice specification for ToolChoice::Function
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct FunctionChoice {
|
||||
pub name: String,
|
||||
}
|
||||
|
||||
/// Tool reference for ToolChoice::AllowedTools
|
||||
///
|
||||
/// Represents a reference to a specific tool in the allowed_tools array.
|
||||
/// Different tool types have different required fields.
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
#[serde(tag = "type")]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum ToolReference {
|
||||
/// Reference to a function tool
|
||||
#[serde(rename = "function")]
|
||||
Function { name: String },
|
||||
|
||||
/// Reference to an MCP tool
|
||||
#[serde(rename = "mcp")]
|
||||
Mcp {
|
||||
server_label: String,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
name: Option<String>,
|
||||
},
|
||||
|
||||
/// File search hosted tool
|
||||
#[serde(rename = "file_search")]
|
||||
FileSearch,
|
||||
|
||||
/// Web search preview hosted tool
|
||||
#[serde(rename = "web_search_preview")]
|
||||
WebSearchPreview,
|
||||
|
||||
/// Computer use preview hosted tool
|
||||
#[serde(rename = "computer_use_preview")]
|
||||
ComputerUsePreview,
|
||||
|
||||
/// Code interpreter hosted tool
|
||||
#[serde(rename = "code_interpreter")]
|
||||
CodeInterpreter,
|
||||
|
||||
/// Image generation hosted tool
|
||||
#[serde(rename = "image_generation")]
|
||||
ImageGeneration,
|
||||
}
|
||||
|
||||
impl ToolReference {
|
||||
/// Get a unique identifier for this tool reference
|
||||
pub fn identifier(&self) -> String {
|
||||
match self {
|
||||
ToolReference::Function { name } => format!("function:{}", name),
|
||||
ToolReference::Mcp { server_label, name } => {
|
||||
if let Some(n) = name {
|
||||
format!("mcp:{}:{}", server_label, n)
|
||||
} else {
|
||||
format!("mcp:{}", server_label)
|
||||
}
|
||||
}
|
||||
ToolReference::FileSearch => "file_search".to_string(),
|
||||
ToolReference::WebSearchPreview => "web_search_preview".to_string(),
|
||||
ToolReference::ComputerUsePreview => "computer_use_preview".to_string(),
|
||||
ToolReference::CodeInterpreter => "code_interpreter".to_string(),
|
||||
ToolReference::ImageGeneration => "image_generation".to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Get the tool name if this is a function tool
|
||||
pub fn function_name(&self) -> Option<&str> {
|
||||
match self {
|
||||
ToolReference::Function { name } => Some(name.as_str()),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct Tool {
|
||||
#[serde(rename = "type")]
|
||||
pub tool_type: String, // "function"
|
||||
pub function: Function,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct Function {
|
||||
pub name: String,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub description: Option<String>,
|
||||
pub parameters: Value, // JSON Schema
|
||||
/// Whether to enable strict schema adherence (OpenAI structured outputs)
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub strict: Option<bool>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct ToolCall {
|
||||
pub id: String,
|
||||
#[serde(rename = "type")]
|
||||
pub tool_type: String, // "function"
|
||||
pub function: FunctionCallResponse,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
#[serde(untagged)]
|
||||
pub enum FunctionCall {
|
||||
None,
|
||||
Auto,
|
||||
Function { name: String },
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct FunctionCallResponse {
|
||||
pub name: String,
|
||||
#[serde(default)]
|
||||
pub arguments: Option<String>, // JSON string
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Usage and Logging
|
||||
// ============================================================================
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct Usage {
|
||||
pub prompt_tokens: u32,
|
||||
pub completion_tokens: u32,
|
||||
pub total_tokens: u32,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub completion_tokens_details: Option<CompletionTokensDetails>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct CompletionTokensDetails {
|
||||
pub reasoning_tokens: Option<u32>,
|
||||
}
|
||||
|
||||
/// Usage information (used by rerank and other endpoints)
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct UsageInfo {
|
||||
pub prompt_tokens: u32,
|
||||
pub completion_tokens: u32,
|
||||
pub total_tokens: u32,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub reasoning_tokens: Option<u32>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub prompt_tokens_details: Option<PromptTokenUsageInfo>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct PromptTokenUsageInfo {
|
||||
pub cached_tokens: u32,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct LogProbs {
|
||||
pub tokens: Vec<String>,
|
||||
pub token_logprobs: Vec<Option<f32>>,
|
||||
pub top_logprobs: Vec<Option<HashMap<String, f32>>>,
|
||||
pub text_offset: Vec<u32>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
#[serde(untagged)]
|
||||
pub enum ChatLogProbs {
|
||||
Detailed {
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
content: Option<Vec<ChatLogProbsContent>>,
|
||||
},
|
||||
Raw(Value),
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct ChatLogProbsContent {
|
||||
pub token: String,
|
||||
pub logprob: f32,
|
||||
pub bytes: Option<Vec<u8>>,
|
||||
pub top_logprobs: Vec<TopLogProb>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct TopLogProb {
|
||||
pub token: String,
|
||||
pub logprob: f32,
|
||||
pub bytes: Option<Vec<u8>>,
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Error Types
|
||||
// ============================================================================
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct ErrorResponse {
|
||||
pub error: ErrorDetail,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct ErrorDetail {
|
||||
pub message: String,
|
||||
#[serde(rename = "type")]
|
||||
pub error_type: String,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub param: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub code: Option<String>,
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Input Types
|
||||
// ============================================================================
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
#[serde(untagged)]
|
||||
pub enum InputIds {
|
||||
Single(Vec<i32>),
|
||||
Batch(Vec<Vec<i32>>),
|
||||
}
|
||||
|
||||
/// LoRA adapter path - can be single path or batch of paths (SGLang extension)
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
#[serde(untagged)]
|
||||
pub enum LoRAPath {
|
||||
Single(Option<String>),
|
||||
Batch(Vec<Option<String>>),
|
||||
}
|
||||
@@ -1,214 +0,0 @@
|
||||
use std::collections::HashMap;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
use super::common::*;
|
||||
|
||||
// ============================================================================
|
||||
// Completions API (v1/completions) - DEPRECATED but still supported
|
||||
// ============================================================================
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct CompletionRequest {
|
||||
/// ID of the model to use (required for OpenAI, optional for some implementations, such as SGLang)
|
||||
pub model: String,
|
||||
|
||||
/// The prompt(s) to generate completions for
|
||||
pub prompt: StringOrArray,
|
||||
|
||||
/// The suffix that comes after a completion of inserted text
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub suffix: Option<String>,
|
||||
|
||||
/// The maximum number of tokens to generate
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub max_tokens: Option<u32>,
|
||||
|
||||
/// What sampling temperature to use, between 0 and 2
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub temperature: Option<f32>,
|
||||
|
||||
/// An alternative to sampling with temperature (nucleus sampling)
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub top_p: Option<f32>,
|
||||
|
||||
/// How many completions to generate for each prompt
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub n: Option<u32>,
|
||||
|
||||
/// Whether to stream back partial progress
|
||||
#[serde(default)]
|
||||
pub stream: bool,
|
||||
|
||||
/// Options for streaming response
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub stream_options: Option<StreamOptions>,
|
||||
|
||||
/// Include the log probabilities on the logprobs most likely tokens
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub logprobs: Option<u32>,
|
||||
|
||||
/// Echo back the prompt in addition to the completion
|
||||
#[serde(default)]
|
||||
pub echo: bool,
|
||||
|
||||
/// Up to 4 sequences where the API will stop generating further tokens
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub stop: Option<StringOrArray>,
|
||||
|
||||
/// Number between -2.0 and 2.0. Positive values penalize new tokens based on whether they appear in the text so far
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub presence_penalty: Option<f32>,
|
||||
|
||||
/// Number between -2.0 and 2.0. Positive values penalize new tokens based on their existing frequency in the text so far
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub frequency_penalty: Option<f32>,
|
||||
|
||||
/// Generates best_of completions server-side and returns the "best"
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub best_of: Option<u32>,
|
||||
|
||||
/// Modify the likelihood of specified tokens appearing in the completion
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub logit_bias: Option<HashMap<String, f32>>,
|
||||
|
||||
/// A unique identifier representing your end-user
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub user: Option<String>,
|
||||
|
||||
/// If specified, our system will make a best effort to sample deterministically
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub seed: Option<i64>,
|
||||
|
||||
// -------- Engine Specific Sampling Parameters --------
|
||||
/// Top-k sampling parameter (-1 to disable)
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub top_k: Option<i32>,
|
||||
|
||||
/// Min-p nucleus sampling parameter
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub min_p: Option<f32>,
|
||||
|
||||
/// Minimum number of tokens to generate
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub min_tokens: Option<u32>,
|
||||
|
||||
/// Repetition penalty for reducing repetitive text
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub repetition_penalty: Option<f32>,
|
||||
|
||||
/// Regex constraint for output generation
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub regex: Option<String>,
|
||||
|
||||
/// EBNF grammar constraint for structured output
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub ebnf: Option<String>,
|
||||
|
||||
/// JSON schema constraint for structured output
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub json_schema: Option<String>,
|
||||
|
||||
/// Specific token IDs to use as stop conditions
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub stop_token_ids: Option<Vec<u32>>,
|
||||
|
||||
/// Skip trimming stop tokens from output
|
||||
#[serde(default)]
|
||||
pub no_stop_trim: bool,
|
||||
|
||||
/// Ignore end-of-sequence tokens during generation
|
||||
#[serde(default)]
|
||||
pub ignore_eos: bool,
|
||||
|
||||
/// Skip special tokens during detokenization
|
||||
#[serde(default = "default_true")]
|
||||
pub skip_special_tokens: bool,
|
||||
|
||||
/// Path to LoRA adapter(s) for model customization
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub lora_path: Option<String>,
|
||||
|
||||
/// Session parameters for continual prompting
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub session_params: Option<HashMap<String, Value>>,
|
||||
|
||||
/// Return model hidden states
|
||||
#[serde(default)]
|
||||
pub return_hidden_states: bool,
|
||||
|
||||
/// Sampling seed for deterministic outputs
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub sampling_seed: Option<u64>,
|
||||
|
||||
/// Additional fields including bootstrap info for PD routing
|
||||
#[serde(flatten)]
|
||||
pub other: Map<String, Value>,
|
||||
}
|
||||
|
||||
impl GenerationRequest for CompletionRequest {
|
||||
fn is_stream(&self) -> bool {
|
||||
self.stream
|
||||
}
|
||||
|
||||
fn get_model(&self) -> Option<&str> {
|
||||
Some(&self.model)
|
||||
}
|
||||
|
||||
fn extract_text_for_routing(&self) -> String {
|
||||
match &self.prompt {
|
||||
StringOrArray::String(s) => s.clone(),
|
||||
StringOrArray::Array(v) => v.join(" "),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Response Types
|
||||
// ============================================================================
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct CompletionResponse {
|
||||
pub id: String,
|
||||
pub object: String, // "text_completion"
|
||||
pub created: u64,
|
||||
pub model: String,
|
||||
pub choices: Vec<CompletionChoice>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub usage: Option<Usage>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub system_fingerprint: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct CompletionChoice {
|
||||
pub text: String,
|
||||
pub index: u32,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub logprobs: Option<LogProbs>,
|
||||
pub finish_reason: Option<String>, // "stop", "length", "content_filter", etc.
|
||||
/// Information about which stop condition was matched
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub matched_stop: Option<Value>, // Can be string or integer
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct CompletionStreamResponse {
|
||||
pub id: String,
|
||||
pub object: String, // "text_completion"
|
||||
pub created: u64,
|
||||
pub choices: Vec<CompletionStreamChoice>,
|
||||
pub model: String,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub system_fingerprint: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct CompletionStreamChoice {
|
||||
pub text: String,
|
||||
pub index: u32,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub logprobs: Option<LogProbs>,
|
||||
pub finish_reason: Option<String>,
|
||||
}
|
||||
@@ -1,76 +0,0 @@
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
|
||||
use super::common::{GenerationRequest, UsageInfo};
|
||||
|
||||
// ============================================================================
|
||||
// Embedding API
|
||||
// ============================================================================
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct EmbeddingRequest {
|
||||
/// ID of the model to use
|
||||
pub model: String,
|
||||
|
||||
/// Input can be a string, array of strings, tokens, or batch inputs
|
||||
pub input: Value,
|
||||
|
||||
/// Optional encoding format (e.g., "float", "base64")
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub encoding_format: Option<String>,
|
||||
|
||||
/// Optional user identifier
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub user: Option<String>,
|
||||
|
||||
/// Optional number of dimensions for the embedding
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub dimensions: Option<u32>,
|
||||
|
||||
/// SGLang extension: request id for tracking
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub rid: Option<String>,
|
||||
|
||||
/// SGLang extension: enable/disable logging of metrics for this request
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub log_metrics: Option<bool>,
|
||||
}
|
||||
|
||||
impl GenerationRequest for EmbeddingRequest {
|
||||
fn is_stream(&self) -> bool {
|
||||
// Embeddings are non-streaming
|
||||
false
|
||||
}
|
||||
|
||||
fn get_model(&self) -> Option<&str> {
|
||||
Some(&self.model)
|
||||
}
|
||||
|
||||
fn extract_text_for_routing(&self) -> String {
|
||||
// Best effort: extract text content for routing decisions
|
||||
match &self.input {
|
||||
Value::String(s) => s.clone(),
|
||||
Value::Array(arr) => arr
|
||||
.iter()
|
||||
.filter_map(|v| v.as_str())
|
||||
.collect::<Vec<_>>()
|
||||
.join(" "),
|
||||
_ => String::new(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct EmbeddingObject {
|
||||
pub object: String, // "embedding"
|
||||
pub embedding: Vec<f32>,
|
||||
pub index: u32,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct EmbeddingResponse {
|
||||
pub object: String, // "list"
|
||||
pub data: Vec<EmbeddingObject>,
|
||||
pub model: String,
|
||||
pub usage: UsageInfo,
|
||||
}
|
||||
@@ -1,228 +0,0 @@
|
||||
use std::fmt;
|
||||
|
||||
/// Response lifecycle events
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
||||
pub enum ResponseEvent {
|
||||
Created,
|
||||
InProgress,
|
||||
Completed,
|
||||
}
|
||||
|
||||
impl ResponseEvent {
|
||||
pub const CREATED: &'static str = "response.created";
|
||||
pub const IN_PROGRESS: &'static str = "response.in_progress";
|
||||
pub const COMPLETED: &'static str = "response.completed";
|
||||
|
||||
pub const fn as_str(&self) -> &'static str {
|
||||
match self {
|
||||
Self::Created => Self::CREATED,
|
||||
Self::InProgress => Self::IN_PROGRESS,
|
||||
Self::Completed => Self::COMPLETED,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for ResponseEvent {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.write_str(self.as_str())
|
||||
}
|
||||
}
|
||||
|
||||
/// Output item events for streaming
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
||||
pub enum OutputItemEvent {
|
||||
Added,
|
||||
Done,
|
||||
Delta,
|
||||
}
|
||||
|
||||
impl OutputItemEvent {
|
||||
pub const ADDED: &'static str = "response.output_item.added";
|
||||
pub const DONE: &'static str = "response.output_item.done";
|
||||
pub const DELTA: &'static str = "response.output_item.delta";
|
||||
|
||||
pub const fn as_str(&self) -> &'static str {
|
||||
match self {
|
||||
Self::Added => Self::ADDED,
|
||||
Self::Done => Self::DONE,
|
||||
Self::Delta => Self::DELTA,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for OutputItemEvent {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.write_str(self.as_str())
|
||||
}
|
||||
}
|
||||
|
||||
/// Function call argument streaming events
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
||||
pub enum FunctionCallEvent {
|
||||
ArgumentsDelta,
|
||||
ArgumentsDone,
|
||||
}
|
||||
|
||||
impl FunctionCallEvent {
|
||||
pub const ARGUMENTS_DELTA: &'static str = "response.function_call_arguments.delta";
|
||||
pub const ARGUMENTS_DONE: &'static str = "response.function_call_arguments.done";
|
||||
|
||||
pub const fn as_str(&self) -> &'static str {
|
||||
match self {
|
||||
Self::ArgumentsDelta => Self::ARGUMENTS_DELTA,
|
||||
Self::ArgumentsDone => Self::ARGUMENTS_DONE,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for FunctionCallEvent {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.write_str(self.as_str())
|
||||
}
|
||||
}
|
||||
|
||||
/// Content part streaming events
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
||||
pub enum ContentPartEvent {
|
||||
Added,
|
||||
Done,
|
||||
}
|
||||
|
||||
impl ContentPartEvent {
|
||||
pub const ADDED: &'static str = "response.content_part.added";
|
||||
pub const DONE: &'static str = "response.content_part.done";
|
||||
|
||||
pub const fn as_str(&self) -> &'static str {
|
||||
match self {
|
||||
Self::Added => Self::ADDED,
|
||||
Self::Done => Self::DONE,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for ContentPartEvent {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.write_str(self.as_str())
|
||||
}
|
||||
}
|
||||
|
||||
/// Output text streaming events
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
||||
pub enum OutputTextEvent {
|
||||
Delta,
|
||||
Done,
|
||||
}
|
||||
|
||||
impl OutputTextEvent {
|
||||
pub const DELTA: &'static str = "response.output_text.delta";
|
||||
pub const DONE: &'static str = "response.output_text.done";
|
||||
|
||||
pub const fn as_str(&self) -> &'static str {
|
||||
match self {
|
||||
Self::Delta => Self::DELTA,
|
||||
Self::Done => Self::DONE,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for OutputTextEvent {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.write_str(self.as_str())
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// MCP Events
|
||||
// ============================================================================
|
||||
|
||||
/// MCP (Model Context Protocol) call events
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
||||
pub enum McpEvent {
|
||||
CallArgumentsDelta,
|
||||
CallArgumentsDone,
|
||||
CallInProgress,
|
||||
CallCompleted,
|
||||
CallFailed,
|
||||
ListToolsInProgress,
|
||||
ListToolsCompleted,
|
||||
}
|
||||
|
||||
impl McpEvent {
|
||||
pub const CALL_ARGUMENTS_DELTA: &'static str = "response.mcp_call_arguments.delta";
|
||||
pub const CALL_ARGUMENTS_DONE: &'static str = "response.mcp_call_arguments.done";
|
||||
pub const CALL_IN_PROGRESS: &'static str = "response.mcp_call.in_progress";
|
||||
pub const CALL_COMPLETED: &'static str = "response.mcp_call.completed";
|
||||
pub const CALL_FAILED: &'static str = "response.mcp_call.failed";
|
||||
pub const LIST_TOOLS_IN_PROGRESS: &'static str = "response.mcp_list_tools.in_progress";
|
||||
pub const LIST_TOOLS_COMPLETED: &'static str = "response.mcp_list_tools.completed";
|
||||
|
||||
pub const fn as_str(&self) -> &'static str {
|
||||
match self {
|
||||
Self::CallArgumentsDelta => Self::CALL_ARGUMENTS_DELTA,
|
||||
Self::CallArgumentsDone => Self::CALL_ARGUMENTS_DONE,
|
||||
Self::CallInProgress => Self::CALL_IN_PROGRESS,
|
||||
Self::CallCompleted => Self::CALL_COMPLETED,
|
||||
Self::CallFailed => Self::CALL_FAILED,
|
||||
Self::ListToolsInProgress => Self::LIST_TOOLS_IN_PROGRESS,
|
||||
Self::ListToolsCompleted => Self::LIST_TOOLS_COMPLETED,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for McpEvent {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.write_str(self.as_str())
|
||||
}
|
||||
}
|
||||
|
||||
/// Item type discriminators used in output items
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
||||
pub enum ItemType {
|
||||
FunctionCall,
|
||||
FunctionToolCall,
|
||||
McpCall,
|
||||
Function,
|
||||
McpListTools,
|
||||
}
|
||||
|
||||
impl ItemType {
|
||||
pub const FUNCTION_CALL: &'static str = "function_call";
|
||||
pub const FUNCTION_TOOL_CALL: &'static str = "function_tool_call";
|
||||
pub const MCP_CALL: &'static str = "mcp_call";
|
||||
pub const FUNCTION: &'static str = "function";
|
||||
pub const MCP_LIST_TOOLS: &'static str = "mcp_list_tools";
|
||||
|
||||
pub const fn as_str(&self) -> &'static str {
|
||||
match self {
|
||||
Self::FunctionCall => Self::FUNCTION_CALL,
|
||||
Self::FunctionToolCall => Self::FUNCTION_TOOL_CALL,
|
||||
Self::McpCall => Self::MCP_CALL,
|
||||
Self::Function => Self::FUNCTION,
|
||||
Self::McpListTools => Self::MCP_LIST_TOOLS,
|
||||
}
|
||||
}
|
||||
|
||||
/// Check if this is a function call variant (FunctionCall or FunctionToolCall)
|
||||
pub const fn is_function_call(&self) -> bool {
|
||||
matches!(self, Self::FunctionCall | Self::FunctionToolCall)
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for ItemType {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.write_str(self.as_str())
|
||||
}
|
||||
}
|
||||
|
||||
/// Check if an event type string matches any response lifecycle event
|
||||
pub fn is_response_event(event_type: &str) -> bool {
|
||||
matches!(
|
||||
event_type,
|
||||
ResponseEvent::CREATED | ResponseEvent::IN_PROGRESS | ResponseEvent::COMPLETED
|
||||
)
|
||||
}
|
||||
|
||||
/// Check if an item type string is a function call variant
|
||||
pub fn is_function_call_type(item_type: &str) -> bool {
|
||||
item_type == ItemType::FUNCTION_CALL || item_type == ItemType::FUNCTION_TOOL_CALL
|
||||
}
|
||||
@@ -1,297 +0,0 @@
|
||||
use std::collections::HashMap;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
use validator::Validate;
|
||||
|
||||
use super::{
|
||||
common::{default_true, GenerationRequest, InputIds},
|
||||
sampling_params::SamplingParams,
|
||||
};
|
||||
use crate::protocols::validated::Normalizable;
|
||||
|
||||
// ============================================================================
|
||||
// SGLang Generate API (native format)
|
||||
// ============================================================================
|
||||
|
||||
#[derive(Clone, Debug, Serialize, Deserialize, Validate)]
|
||||
#[validate(schema(function = "validate_generate_request"))]
|
||||
pub struct GenerateRequest {
|
||||
/// Text input - SGLang native format
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub text: Option<String>,
|
||||
|
||||
pub model: Option<String>,
|
||||
|
||||
/// Input IDs for tokenized input
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub input_ids: Option<InputIds>,
|
||||
|
||||
/// Input embeddings for direct embedding input
|
||||
/// Can be a 2D array (single request) or 3D array (batch of requests)
|
||||
/// Placeholder for future use
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub input_embeds: Option<Value>,
|
||||
|
||||
/// Image input data
|
||||
/// Can be an image instance, file name, URL, or base64 encoded string
|
||||
/// Supports single images, lists of images, or nested lists for batch processing
|
||||
/// Placeholder for future use
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub image_data: Option<Value>,
|
||||
|
||||
/// Video input data
|
||||
/// Can be a file name, URL, or base64 encoded string
|
||||
/// Supports single videos, lists of videos, or nested lists for batch processing
|
||||
/// Placeholder for future use
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub video_data: Option<Value>,
|
||||
|
||||
/// Audio input data
|
||||
/// Can be a file name, URL, or base64 encoded string
|
||||
/// Supports single audio files, lists of audio, or nested lists for batch processing
|
||||
/// Placeholder for future use
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub audio_data: Option<Value>,
|
||||
|
||||
/// Sampling parameters (sglang style)
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub sampling_params: Option<SamplingParams>,
|
||||
|
||||
/// Whether to return logprobs
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub return_logprob: Option<bool>,
|
||||
|
||||
/// If return logprobs, the start location in the prompt for returning logprobs.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub logprob_start_len: Option<i32>,
|
||||
|
||||
/// If return logprobs, the number of top logprobs to return at each position.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub top_logprobs_num: Option<i32>,
|
||||
|
||||
/// If return logprobs, the token ids to return logprob for.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub token_ids_logprob: Option<Vec<u32>>,
|
||||
|
||||
/// Whether to detokenize tokens in text in the returned logprobs.
|
||||
#[serde(default)]
|
||||
pub return_text_in_logprobs: bool,
|
||||
|
||||
/// Whether to stream the response
|
||||
#[serde(default)]
|
||||
pub stream: bool,
|
||||
|
||||
/// Whether to log metrics for this request (e.g. health_generate calls do not log metrics)
|
||||
#[serde(default = "default_true")]
|
||||
pub log_metrics: bool,
|
||||
|
||||
/// Return model hidden states
|
||||
#[serde(default)]
|
||||
pub return_hidden_states: bool,
|
||||
|
||||
/// The modalities of the image data [image, multi-images, video]
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub modalities: Option<Vec<String>>,
|
||||
|
||||
/// Session parameters for continual prompting
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub session_params: Option<HashMap<String, Value>>,
|
||||
|
||||
/// Path to LoRA adapter(s) for model customization
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub lora_path: Option<String>,
|
||||
|
||||
/// LoRA adapter ID (if pre-loaded)
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub lora_id: Option<String>,
|
||||
|
||||
/// Custom logit processor for advanced sampling control. Must be a serialized instance
|
||||
/// of `CustomLogitProcessor` in python/sglang/srt/sampling/custom_logit_processor.py
|
||||
/// Use the processor's `to_str()` method to generate the serialized string.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub custom_logit_processor: Option<String>,
|
||||
|
||||
/// For disaggregated inference
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub bootstrap_host: Option<String>,
|
||||
|
||||
/// For disaggregated inference
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub bootstrap_port: Option<i32>,
|
||||
|
||||
/// For disaggregated inference
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub bootstrap_room: Option<i32>,
|
||||
|
||||
/// For disaggregated inference
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub bootstrap_pair_key: Option<String>,
|
||||
|
||||
/// Data parallel rank routing
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub data_parallel_rank: Option<i32>,
|
||||
|
||||
/// Background response
|
||||
#[serde(default)]
|
||||
pub background: bool,
|
||||
|
||||
/// Conversation ID for tracking
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub conversation_id: Option<String>,
|
||||
|
||||
/// Priority for the request
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub priority: Option<i32>,
|
||||
|
||||
/// Extra key for classifying the request (e.g. cache_salt)
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub extra_key: Option<String>,
|
||||
|
||||
/// Whether to disallow logging for this request (e.g. due to ZDR)
|
||||
#[serde(default)]
|
||||
pub no_logs: bool,
|
||||
|
||||
/// Custom metric labels
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub custom_labels: Option<HashMap<String, String>>,
|
||||
|
||||
/// Whether to return bytes for image generation
|
||||
#[serde(default)]
|
||||
pub return_bytes: bool,
|
||||
|
||||
/// Whether to return entropy
|
||||
#[serde(default)]
|
||||
pub return_entropy: bool,
|
||||
|
||||
/// Request ID for tracking (inherited from BaseReq in Python)
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub rid: Option<String>,
|
||||
}
|
||||
|
||||
impl Normalizable for GenerateRequest {
|
||||
// Use default no-op implementation - no normalization needed for GenerateRequest
|
||||
}
|
||||
|
||||
/// Validation function for GenerateRequest - ensure exactly one input type is provided
|
||||
fn validate_generate_request(req: &GenerateRequest) -> Result<(), validator::ValidationError> {
|
||||
// Exactly one of text or input_ids must be provided
|
||||
// Note: input_embeds not yet supported in Rust implementation
|
||||
let has_text = req.text.is_some();
|
||||
let has_input_ids = req.input_ids.is_some();
|
||||
|
||||
let count = [has_text, has_input_ids].iter().filter(|&&x| x).count();
|
||||
|
||||
if count == 0 {
|
||||
return Err(validator::ValidationError::new(
|
||||
"Either text or input_ids should be provided.",
|
||||
));
|
||||
}
|
||||
|
||||
if count > 1 {
|
||||
return Err(validator::ValidationError::new(
|
||||
"Either text or input_ids should be provided.",
|
||||
));
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
impl GenerationRequest for GenerateRequest {
|
||||
fn is_stream(&self) -> bool {
|
||||
self.stream
|
||||
}
|
||||
|
||||
fn get_model(&self) -> Option<&str> {
|
||||
// Generate requests have an optional model field
|
||||
if let Some(s) = &self.model {
|
||||
Some(s.as_str())
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
fn extract_text_for_routing(&self) -> String {
|
||||
// Check fields in priority order: text, input_ids
|
||||
if let Some(ref text) = self.text {
|
||||
return text.clone();
|
||||
}
|
||||
|
||||
if let Some(ref input_ids) = self.input_ids {
|
||||
return match input_ids {
|
||||
InputIds::Single(ids) => ids
|
||||
.iter()
|
||||
.map(|&id| id.to_string())
|
||||
.collect::<Vec<String>>()
|
||||
.join(" "),
|
||||
InputIds::Batch(batches) => batches
|
||||
.iter()
|
||||
.flat_map(|batch| batch.iter().map(|&id| id.to_string()))
|
||||
.collect::<Vec<String>>()
|
||||
.join(" "),
|
||||
};
|
||||
}
|
||||
|
||||
// No text input found
|
||||
String::new()
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// SGLang Generate Response Types
|
||||
// ============================================================================
|
||||
|
||||
/// SGLang generate response (single completion or array for n>1)
|
||||
///
|
||||
/// Format for n=1:
|
||||
/// ```json
|
||||
/// {
|
||||
/// "text": "...",
|
||||
/// "output_ids": [...],
|
||||
/// "meta_info": { ... }
|
||||
/// }
|
||||
/// ```
|
||||
///
|
||||
/// Format for n>1:
|
||||
/// ```json
|
||||
/// [
|
||||
/// {"text": "...", "output_ids": [...], "meta_info": {...}},
|
||||
/// {"text": "...", "output_ids": [...], "meta_info": {...}}
|
||||
/// ]
|
||||
/// ```
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct GenerateResponse {
|
||||
pub text: String,
|
||||
pub output_ids: Vec<u32>,
|
||||
pub meta_info: GenerateMetaInfo,
|
||||
}
|
||||
|
||||
/// Metadata for a single generate completion
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct GenerateMetaInfo {
|
||||
pub id: String,
|
||||
pub finish_reason: GenerateFinishReason,
|
||||
pub prompt_tokens: u32,
|
||||
pub weight_version: String,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub input_token_logprobs: Option<Vec<Vec<Option<f64>>>>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub output_token_logprobs: Option<Vec<Vec<Option<f64>>>>,
|
||||
pub completion_tokens: u32,
|
||||
pub cached_tokens: u32,
|
||||
pub e2e_latency: f64,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub matched_stop: Option<Value>,
|
||||
}
|
||||
|
||||
/// Finish reason for generate endpoint
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(tag = "type", rename_all = "lowercase")]
|
||||
pub enum GenerateFinishReason {
|
||||
Length {
|
||||
length: u32,
|
||||
},
|
||||
Stop,
|
||||
#[serde(untagged)]
|
||||
Other(Value),
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,25 +0,0 @@
|
||||
// Protocol definitions and validation for various LLM APIs
|
||||
// This module provides a structured approach to handling different API protocols
|
||||
|
||||
/// Default model identifier used when no model is specified.
|
||||
///
|
||||
/// This constant should be used instead of hardcoded "unknown" strings
|
||||
/// throughout the codebase for consistency.
|
||||
pub const UNKNOWN_MODEL_ID: &str = "unknown";
|
||||
|
||||
pub mod builders;
|
||||
pub mod chat;
|
||||
pub mod classify;
|
||||
pub mod common;
|
||||
pub mod completion;
|
||||
pub mod embedding;
|
||||
pub mod event_types;
|
||||
pub mod generate;
|
||||
pub mod messages;
|
||||
pub mod parser;
|
||||
pub mod rerank;
|
||||
pub mod responses;
|
||||
pub mod sampling_params;
|
||||
pub mod tokenize;
|
||||
pub mod validated;
|
||||
pub mod worker_spec;
|
||||
@@ -1,23 +0,0 @@
|
||||
use serde::Deserialize;
|
||||
|
||||
use crate::protocols::common::Tool;
|
||||
|
||||
/// Request to parse function calls from model output text
|
||||
#[derive(Deserialize)]
|
||||
pub struct ParseFunctionCallRequest {
|
||||
/// The text to parse for function calls
|
||||
pub text: String,
|
||||
/// The parser type/name to use for parsing (e.g., "json", "pythonic")
|
||||
pub tool_call_parser: String,
|
||||
/// The list of available tools that the model can call
|
||||
pub tools: Vec<Tool>,
|
||||
}
|
||||
|
||||
/// Request to separate reasoning from normal text in model output
|
||||
#[derive(Deserialize)]
|
||||
pub struct SeparateReasoningRequest {
|
||||
/// The text to parse for reasoning content
|
||||
pub text: String,
|
||||
/// The parser type/name to use for reasoning detection (e.g., "step3", "deepseek_r1")
|
||||
pub reasoning_parser: String,
|
||||
}
|
||||
@@ -1,212 +0,0 @@
|
||||
use std::collections::HashMap;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
use validator::Validate;
|
||||
|
||||
use super::common::{default_model, default_true, GenerationRequest, StringOrArray, UsageInfo};
|
||||
|
||||
fn default_rerank_object() -> String {
|
||||
"rerank".to_string()
|
||||
}
|
||||
|
||||
/// TODO: Create timestamp should not be in protocol layer
|
||||
fn current_timestamp() -> i64 {
|
||||
std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap_or_else(|_| std::time::Duration::from_secs(0))
|
||||
.as_secs() as i64
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Rerank API
|
||||
// ============================================================================
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize, Validate)]
|
||||
#[validate(schema(function = "validate_rerank_request"))]
|
||||
pub struct RerankRequest {
|
||||
/// The query text to rank documents against
|
||||
#[validate(custom(function = "validate_query"))]
|
||||
pub query: String,
|
||||
|
||||
/// List of documents to be ranked
|
||||
#[validate(custom(function = "validate_documents"))]
|
||||
pub documents: Vec<String>,
|
||||
|
||||
/// Model to use for reranking
|
||||
#[serde(default = "default_model")]
|
||||
pub model: String,
|
||||
|
||||
/// Maximum number of documents to return (optional)
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
#[validate(range(min = 1))]
|
||||
pub top_k: Option<usize>,
|
||||
|
||||
/// Whether to return documents in addition to scores
|
||||
#[serde(default = "default_true")]
|
||||
pub return_documents: bool,
|
||||
|
||||
// SGLang specific extensions
|
||||
/// Request ID for tracking
|
||||
pub rid: Option<StringOrArray>,
|
||||
|
||||
/// User identifier
|
||||
pub user: Option<String>,
|
||||
}
|
||||
|
||||
impl GenerationRequest for RerankRequest {
|
||||
fn get_model(&self) -> Option<&str> {
|
||||
Some(&self.model)
|
||||
}
|
||||
|
||||
fn is_stream(&self) -> bool {
|
||||
false // Reranking doesn't support streaming
|
||||
}
|
||||
|
||||
fn extract_text_for_routing(&self) -> String {
|
||||
self.query.clone()
|
||||
}
|
||||
}
|
||||
|
||||
impl super::validated::Normalizable for RerankRequest {
|
||||
// Use default no-op normalization
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Validation Functions
|
||||
// ============================================================================
|
||||
|
||||
/// Validates that the query is not empty
|
||||
fn validate_query(query: &str) -> Result<(), validator::ValidationError> {
|
||||
if query.trim().is_empty() {
|
||||
return Err(validator::ValidationError::new("query cannot be empty"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Validates that the documents list is not empty
|
||||
fn validate_documents(documents: &[String]) -> Result<(), validator::ValidationError> {
|
||||
if documents.is_empty() {
|
||||
return Err(validator::ValidationError::new(
|
||||
"documents list cannot be empty",
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Schema-level validation for cross-field dependencies
|
||||
fn validate_rerank_request(req: &RerankRequest) -> Result<(), validator::ValidationError> {
|
||||
// Validate top_k if specified
|
||||
if let Some(k) = req.top_k {
|
||||
if k > req.documents.len() {
|
||||
// This is allowed but we log a warning
|
||||
tracing::warn!(
|
||||
"top_k ({}) is greater than number of documents ({})",
|
||||
k,
|
||||
req.documents.len()
|
||||
);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
impl RerankRequest {
|
||||
/// Get the effective top_k value
|
||||
pub fn effective_top_k(&self) -> usize {
|
||||
self.top_k.unwrap_or(self.documents.len())
|
||||
}
|
||||
}
|
||||
|
||||
/// Individual rerank result
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct RerankResult {
|
||||
/// Relevance score for the document
|
||||
pub score: f32,
|
||||
|
||||
/// The document text (if return_documents was true)
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub document: Option<String>,
|
||||
|
||||
/// Original index of the document in the request
|
||||
pub index: usize,
|
||||
|
||||
/// Additional metadata about the ranking
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub meta_info: Option<HashMap<String, Value>>,
|
||||
}
|
||||
|
||||
/// Rerank response containing sorted results
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct RerankResponse {
|
||||
/// Ranked results sorted by score (highest first)
|
||||
pub results: Vec<RerankResult>,
|
||||
|
||||
/// Model used for reranking
|
||||
pub model: String,
|
||||
|
||||
/// Usage information
|
||||
pub usage: Option<UsageInfo>,
|
||||
|
||||
/// Response object type
|
||||
#[serde(default = "default_rerank_object")]
|
||||
pub object: String,
|
||||
|
||||
/// Response ID
|
||||
pub id: Option<StringOrArray>,
|
||||
|
||||
/// Creation timestamp
|
||||
pub created: i64,
|
||||
}
|
||||
|
||||
impl RerankResponse {
|
||||
/// Create a new RerankResponse with the given results and model
|
||||
pub fn new(
|
||||
results: Vec<RerankResult>,
|
||||
model: String,
|
||||
request_id: Option<StringOrArray>,
|
||||
) -> Self {
|
||||
RerankResponse {
|
||||
results,
|
||||
model,
|
||||
usage: None,
|
||||
object: default_rerank_object(),
|
||||
id: request_id,
|
||||
created: current_timestamp(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Apply top_k limit to results
|
||||
pub fn apply_top_k(&mut self, k: usize) {
|
||||
self.results.truncate(k);
|
||||
}
|
||||
|
||||
/// Drop documents from results (when return_documents is false)
|
||||
pub fn drop_documents(&mut self) {
|
||||
for result in &mut self.results {
|
||||
result.document = None;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// V1 API compatibility format for rerank requests
|
||||
/// Matches Python's V1RerankReqInput
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct V1RerankReqInput {
|
||||
pub query: String,
|
||||
pub documents: Vec<String>,
|
||||
}
|
||||
|
||||
/// Convert V1RerankReqInput to RerankRequest
|
||||
impl From<V1RerankReqInput> for RerankRequest {
|
||||
fn from(v1: V1RerankReqInput) -> Self {
|
||||
RerankRequest {
|
||||
query: v1.query,
|
||||
documents: v1.documents,
|
||||
model: default_model(),
|
||||
top_k: None,
|
||||
return_documents: true,
|
||||
rid: None,
|
||||
user: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,119 +0,0 @@
|
||||
use serde::{Deserialize, Serialize};
|
||||
use validator::Validate;
|
||||
|
||||
use super::common::StringOrArray;
|
||||
|
||||
/// Sampling parameters for text generation
|
||||
#[derive(Debug, Clone, Deserialize, Serialize, Default, Validate)]
|
||||
#[validate(schema(function = "validate_sampling_params"))]
|
||||
pub struct SamplingParams {
|
||||
/// Temperature for sampling (must be >= 0.0, no upper limit)
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
#[validate(range(min = 0.0))]
|
||||
pub temperature: Option<f32>,
|
||||
/// Maximum number of new tokens to generate (must be >= 0)
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
#[validate(range(min = 0))]
|
||||
pub max_new_tokens: Option<u32>,
|
||||
/// Top-p nucleus sampling (0.0 < top_p <= 1.0)
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
#[validate(custom(function = "validate_top_p_value"))]
|
||||
pub top_p: Option<f32>,
|
||||
/// Top-k sampling (-1 to disable, or >= 1)
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
#[validate(custom(function = "validate_top_k_value"))]
|
||||
pub top_k: Option<i32>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
#[validate(range(min = -2.0, max = 2.0))]
|
||||
pub frequency_penalty: Option<f32>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
#[validate(range(min = -2.0, max = 2.0))]
|
||||
pub presence_penalty: Option<f32>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
#[validate(range(min = 0.0, max = 2.0))]
|
||||
pub repetition_penalty: Option<f32>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub stop: Option<StringOrArray>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub ignore_eos: Option<bool>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub skip_special_tokens: Option<bool>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub json_schema: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub regex: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub ebnf: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
#[validate(range(min = 0.0, max = 1.0))]
|
||||
pub min_p: Option<f32>,
|
||||
/// Minimum number of new tokens (validated in schema function for cross-field check with max_new_tokens)
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub min_new_tokens: Option<u32>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub stop_token_ids: Option<Vec<u32>>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub no_stop_trim: Option<bool>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub n: Option<u32>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub sampling_seed: Option<u64>,
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Shared Validation Functions
|
||||
// ============================================================================
|
||||
|
||||
/// Validates top_p: 0.0 < top_p <= 1.0 (can't use range validator for open interval)
|
||||
pub fn validate_top_p_value(top_p: f32) -> Result<(), validator::ValidationError> {
|
||||
if !(top_p > 0.0 && top_p <= 1.0) {
|
||||
return Err(validator::ValidationError::new(
|
||||
"top_p must be in (0, 1] - greater than 0.0 and at most 1.0",
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Validates top_k: -1 (disabled) or >= 1 (special -1 case - can't use range validator)
|
||||
pub fn validate_top_k_value(top_k: i32) -> Result<(), validator::ValidationError> {
|
||||
if top_k != -1 && top_k < 1 {
|
||||
return Err(validator::ValidationError::new(
|
||||
"top_k must be -1 (disabled) or at least 1",
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// SamplingParams-Specific Validation
|
||||
// ============================================================================
|
||||
|
||||
/// Validation function for SamplingParams - cross-field validation only
|
||||
fn validate_sampling_params(params: &SamplingParams) -> Result<(), validator::ValidationError> {
|
||||
// 1. Cross-field validation: min_new_tokens <= max_new_tokens
|
||||
if let (Some(min), Some(max)) = (params.min_new_tokens, params.max_new_tokens) {
|
||||
if min > max {
|
||||
return Err(validator::ValidationError::new(
|
||||
"min_new_tokens cannot exceed max_new_tokens",
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
// 2. Validate mutually exclusive structured output constraints
|
||||
let constraint_count = [
|
||||
params.regex.is_some(),
|
||||
params.ebnf.is_some(),
|
||||
params.json_schema.is_some(),
|
||||
]
|
||||
.iter()
|
||||
.filter(|&&x| x)
|
||||
.count();
|
||||
|
||||
if constraint_count > 1 {
|
||||
return Err(validator::ValidationError::new(
|
||||
"only one of regex, ebnf, or json_schema can be set",
|
||||
));
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -1,279 +0,0 @@
|
||||
//! Tokenize and Detokenize API protocol types
|
||||
//!
|
||||
//! These types mirror the SGLang Python implementation for compatibility.
|
||||
//! See: python/sglang/srt/entrypoints/openai/protocol.py
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use super::UNKNOWN_MODEL_ID;
|
||||
|
||||
// ============================================================================
|
||||
// Tokenize API
|
||||
// ============================================================================
|
||||
|
||||
/// Request schema for the /v1/tokenize endpoint
|
||||
///
|
||||
/// Supports both single string and batch tokenization.
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct TokenizeRequest {
|
||||
/// Model name for tokenizer selection
|
||||
#[serde(default = "default_model_name")]
|
||||
pub model: String,
|
||||
|
||||
/// Text(s) to tokenize - can be a single string or array of strings
|
||||
pub prompt: StringOrArray,
|
||||
}
|
||||
|
||||
/// Response schema for the /v1/tokenize endpoint
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct TokenizeResponse {
|
||||
/// Token IDs - single list for single input, nested list for batch
|
||||
pub tokens: TokensResult,
|
||||
|
||||
/// Token count(s) - single int for single input, list for batch
|
||||
pub count: CountResult,
|
||||
|
||||
/// Character count(s) of input - single int for single input, list for batch
|
||||
pub char_count: CountResult,
|
||||
}
|
||||
|
||||
/// Token IDs result - either single or batch
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(untagged)]
|
||||
pub enum TokensResult {
|
||||
Single(Vec<u32>),
|
||||
Batch(Vec<Vec<u32>>),
|
||||
}
|
||||
|
||||
/// Count result - either single or batch
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(untagged)]
|
||||
pub enum CountResult {
|
||||
Single(i32),
|
||||
Batch(Vec<i32>),
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Detokenize API
|
||||
// ============================================================================
|
||||
|
||||
/// Request schema for the /v1/detokenize endpoint
|
||||
///
|
||||
/// Supports both single sequence and batch detokenization.
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct DetokenizeRequest {
|
||||
/// Model name for tokenizer selection
|
||||
#[serde(default = "default_model_name")]
|
||||
pub model: String,
|
||||
|
||||
/// Token IDs to detokenize - single list or batch (list of lists)
|
||||
pub tokens: TokensInput,
|
||||
|
||||
/// Whether to skip special tokens (e.g., padding or EOS) during decoding
|
||||
#[serde(default = "default_true")]
|
||||
pub skip_special_tokens: bool,
|
||||
}
|
||||
|
||||
/// Token input - either single sequence or batch
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
#[serde(untagged)]
|
||||
pub enum TokensInput {
|
||||
/// Single sequence of token IDs
|
||||
Single(Vec<u32>),
|
||||
/// Batch of token sequences
|
||||
Batch(Vec<Vec<u32>>),
|
||||
}
|
||||
|
||||
impl TokensInput {
|
||||
/// Check if this is a batch input
|
||||
pub fn is_batch(&self) -> bool {
|
||||
matches!(self, TokensInput::Batch(_))
|
||||
}
|
||||
|
||||
/// Get the sequences (always returns a vec of vecs for uniform processing)
|
||||
pub fn sequences(&self) -> Vec<&[u32]> {
|
||||
match self {
|
||||
TokensInput::Single(seq) => vec![seq.as_slice()],
|
||||
TokensInput::Batch(seqs) => seqs.iter().map(|s| s.as_slice()).collect(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Response schema for the /v1/detokenize endpoint
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct DetokenizeResponse {
|
||||
/// Decoded text - single string for single input, list for batch
|
||||
pub text: TextResult,
|
||||
}
|
||||
|
||||
/// Text result - either single or batch
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(untagged)]
|
||||
pub enum TextResult {
|
||||
Single(String),
|
||||
Batch(Vec<String>),
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Tokenizer Management API
|
||||
// ============================================================================
|
||||
|
||||
/// Request schema for adding a tokenizer
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct AddTokenizerRequest {
|
||||
/// Name to register the tokenizer under
|
||||
pub name: String,
|
||||
|
||||
/// Source: either a local path or HuggingFace model ID
|
||||
pub source: String,
|
||||
|
||||
/// Optional path to chat template file
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub chat_template_path: Option<String>,
|
||||
}
|
||||
|
||||
/// Response schema for adding a tokenizer (async)
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct AddTokenizerResponse {
|
||||
/// Unique identifier for the tokenizer (UUID)
|
||||
pub id: String,
|
||||
/// Status of the request: "pending", "processing", "completed", "failed"
|
||||
pub status: String,
|
||||
pub message: String,
|
||||
/// Vocabulary size of the loaded tokenizer (only set on completion)
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub vocab_size: Option<usize>,
|
||||
}
|
||||
|
||||
/// Response schema for listing tokenizers
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ListTokenizersResponse {
|
||||
pub tokenizers: Vec<TokenizerInfo>,
|
||||
}
|
||||
|
||||
/// Information about a registered tokenizer
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct TokenizerInfo {
|
||||
/// Unique identifier (UUID)
|
||||
pub id: String,
|
||||
/// User-provided name
|
||||
pub name: String,
|
||||
/// Source path or HuggingFace model ID
|
||||
pub source: String,
|
||||
pub vocab_size: usize,
|
||||
}
|
||||
|
||||
/// Request schema for removing a tokenizer
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct RemoveTokenizerRequest {
|
||||
/// Name of the tokenizer to remove
|
||||
pub name: String,
|
||||
}
|
||||
|
||||
/// Response schema for removing a tokenizer
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct RemoveTokenizerResponse {
|
||||
pub success: bool,
|
||||
pub message: String,
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Helper Types
|
||||
// ============================================================================
|
||||
|
||||
/// String or array of strings (for flexible input)
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
#[serde(untagged)]
|
||||
pub enum StringOrArray {
|
||||
Single(String),
|
||||
Array(Vec<String>),
|
||||
}
|
||||
|
||||
impl StringOrArray {
|
||||
/// Check if this is a batch (array) input
|
||||
pub fn is_batch(&self) -> bool {
|
||||
matches!(self, StringOrArray::Array(_))
|
||||
}
|
||||
|
||||
/// Get all strings as a slice (converts single to vec)
|
||||
pub fn as_strings(&self) -> Vec<&str> {
|
||||
match self {
|
||||
StringOrArray::Single(s) => vec![s.as_str()],
|
||||
StringOrArray::Array(arr) => arr.iter().map(|s| s.as_str()).collect(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Default Functions
|
||||
// ============================================================================
|
||||
|
||||
fn default_model_name() -> String {
|
||||
UNKNOWN_MODEL_ID.to_string()
|
||||
}
|
||||
|
||||
fn default_true() -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_tokenize_request_single() {
|
||||
let json = r#"{"prompt": "Hello world"}"#;
|
||||
let req: TokenizeRequest = serde_json::from_str(json).unwrap();
|
||||
assert_eq!(req.model, "unknown");
|
||||
assert!(matches!(req.prompt, StringOrArray::Single(_)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_tokenize_request_batch() {
|
||||
let json = r#"{"model": "llama", "prompt": ["Hello", "World"]}"#;
|
||||
let req: TokenizeRequest = serde_json::from_str(json).unwrap();
|
||||
assert_eq!(req.model, "llama");
|
||||
assert!(matches!(req.prompt, StringOrArray::Array(_)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_detokenize_request_single() {
|
||||
let json = r#"{"tokens": [1, 2, 3]}"#;
|
||||
let req: DetokenizeRequest = serde_json::from_str(json).unwrap();
|
||||
assert!(matches!(req.tokens, TokensInput::Single(_)));
|
||||
assert!(req.skip_special_tokens);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_detokenize_request_batch() {
|
||||
let json = r#"{"tokens": [[1, 2], [3, 4, 5]], "skip_special_tokens": false}"#;
|
||||
let req: DetokenizeRequest = serde_json::from_str(json).unwrap();
|
||||
assert!(matches!(req.tokens, TokensInput::Batch(_)));
|
||||
assert!(!req.skip_special_tokens);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_tokenize_response_single() {
|
||||
let resp = TokenizeResponse {
|
||||
tokens: TokensResult::Single(vec![1, 2, 3]),
|
||||
count: CountResult::Single(3),
|
||||
char_count: CountResult::Single(11),
|
||||
};
|
||||
let json = serde_json::to_string(&resp).unwrap();
|
||||
assert!(json.contains("[1,2,3]"));
|
||||
assert!(json.contains("\"count\":3"));
|
||||
assert!(json.contains("\"char_count\":11"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_tokenize_response_batch() {
|
||||
let resp = TokenizeResponse {
|
||||
tokens: TokensResult::Batch(vec![vec![1, 2], vec![3, 4, 5]]),
|
||||
count: CountResult::Batch(vec![2, 3]),
|
||||
char_count: CountResult::Batch(vec![5, 5]),
|
||||
};
|
||||
let json = serde_json::to_string(&resp).unwrap();
|
||||
assert!(json.contains("[[1,2],[3,4,5]]"));
|
||||
assert!(json.contains("[2,3]"));
|
||||
}
|
||||
}
|
||||
@@ -1,164 +0,0 @@
|
||||
// Validated JSON extractor for automatic request validation
|
||||
//
|
||||
// This module provides a ValidatedJson extractor that automatically validates
|
||||
// requests using the validator crate's Validate trait.
|
||||
|
||||
use axum::{
|
||||
extract::{rejection::JsonRejection, FromRequest, Request},
|
||||
http::StatusCode,
|
||||
response::{IntoResponse, Response},
|
||||
Json,
|
||||
};
|
||||
use serde::de::DeserializeOwned;
|
||||
use serde_json::json;
|
||||
use validator::Validate;
|
||||
|
||||
/// Trait for request types that need post-deserialization normalization
|
||||
pub trait Normalizable {
|
||||
/// Normalize the request by applying defaults and transformations
|
||||
fn normalize(&mut self) {
|
||||
// Default: no-op
|
||||
}
|
||||
}
|
||||
|
||||
/// A JSON extractor that automatically validates and normalizes the request body
|
||||
///
|
||||
/// This extractor deserializes the request body and automatically calls `.validate()`
|
||||
/// on types that implement the `Validate` trait. If validation fails, it returns
|
||||
/// a 400 Bad Request with detailed error information.
|
||||
///
|
||||
/// # Example
|
||||
///
|
||||
/// ```rust,ignore
|
||||
/// async fn create_chat(
|
||||
/// ValidatedJson(request): ValidatedJson<ChatCompletionRequest>,
|
||||
/// ) -> Response {
|
||||
/// // request is guaranteed to be valid here
|
||||
/// process_request(request).await
|
||||
/// }
|
||||
/// ```
|
||||
pub struct ValidatedJson<T>(pub T);
|
||||
|
||||
impl<S, T> FromRequest<S> for ValidatedJson<T>
|
||||
where
|
||||
T: DeserializeOwned + Validate + Normalizable + Send,
|
||||
S: Send + Sync,
|
||||
{
|
||||
type Rejection = Response;
|
||||
|
||||
async fn from_request(req: Request, state: &S) -> Result<Self, Self::Rejection> {
|
||||
// First, extract and deserialize the JSON
|
||||
let Json(mut data) =
|
||||
Json::<T>::from_request(req, state)
|
||||
.await
|
||||
.map_err(|err: JsonRejection| {
|
||||
let error_message = match err {
|
||||
JsonRejection::JsonDataError(e) => {
|
||||
format!("Invalid JSON data: {}", e)
|
||||
}
|
||||
JsonRejection::JsonSyntaxError(e) => {
|
||||
format!("JSON syntax error: {}", e)
|
||||
}
|
||||
JsonRejection::MissingJsonContentType(_) => {
|
||||
"Missing Content-Type: application/json header".to_string()
|
||||
}
|
||||
_ => format!("Failed to parse JSON: {}", err),
|
||||
};
|
||||
|
||||
(
|
||||
StatusCode::BAD_REQUEST,
|
||||
Json(json!({
|
||||
"error": {
|
||||
"message": error_message,
|
||||
"type": "invalid_request_error",
|
||||
"code": "json_parse_error"
|
||||
}
|
||||
})),
|
||||
)
|
||||
.into_response()
|
||||
})?;
|
||||
|
||||
// Normalize the request (apply defaults based on other fields)
|
||||
data.normalize();
|
||||
|
||||
// Then, automatically validate the data
|
||||
data.validate().map_err(|validation_errors| {
|
||||
(
|
||||
StatusCode::BAD_REQUEST,
|
||||
Json(json!({
|
||||
"error": {
|
||||
"message": validation_errors.to_string(),
|
||||
"type": "invalid_request_error",
|
||||
"code": 400
|
||||
}
|
||||
})),
|
||||
)
|
||||
.into_response()
|
||||
})?;
|
||||
|
||||
Ok(ValidatedJson(data))
|
||||
}
|
||||
}
|
||||
|
||||
// Implement Deref to allow transparent access to the inner value
|
||||
impl<T> std::ops::Deref for ValidatedJson<T> {
|
||||
type Target = T;
|
||||
|
||||
fn deref(&self) -> &Self::Target {
|
||||
&self.0
|
||||
}
|
||||
}
|
||||
|
||||
impl<T> std::ops::DerefMut for ValidatedJson<T> {
|
||||
fn deref_mut(&mut self) -> &mut Self::Target {
|
||||
&mut self.0
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use serde::{Deserialize, Serialize};
|
||||
use validator::Validate;
|
||||
|
||||
use super::*;
|
||||
|
||||
#[derive(Debug, Deserialize, Serialize, Validate)]
|
||||
struct TestRequest {
|
||||
#[validate(range(min = 0.0, max = 1.0))]
|
||||
value: f32,
|
||||
#[validate(length(min = 1))]
|
||||
name: String,
|
||||
}
|
||||
|
||||
impl Normalizable for TestRequest {
|
||||
// Use default no-op implementation
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_validated_json_valid() {
|
||||
// This test is conceptual - actual testing would require Axum test harness
|
||||
let request = TestRequest {
|
||||
value: 0.5,
|
||||
name: "test".to_string(),
|
||||
};
|
||||
assert!(request.validate().is_ok());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_validated_json_invalid_range() {
|
||||
let request = TestRequest {
|
||||
value: 1.5, // Out of range
|
||||
name: "test".to_string(),
|
||||
};
|
||||
assert!(request.validate().is_err());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_validated_json_invalid_length() {
|
||||
let request = TestRequest {
|
||||
value: 0.5,
|
||||
name: "".to_string(), // Empty name
|
||||
};
|
||||
assert!(request.validate().is_err());
|
||||
}
|
||||
}
|
||||
@@ -1,375 +0,0 @@
|
||||
//! Worker management API specifications
|
||||
//!
|
||||
//! Defines the request/response structures for worker management endpoints
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use super::UNKNOWN_MODEL_ID;
|
||||
|
||||
/// Worker configuration for API requests
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct WorkerConfigRequest {
|
||||
/// Worker URL (required)
|
||||
pub url: String,
|
||||
|
||||
/// Worker API key (optional)
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub api_key: Option<String>,
|
||||
|
||||
/// Model ID (optional, will query from server if not provided)
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub model_id: Option<String>,
|
||||
|
||||
/// Worker priority (optional, default: 50, higher = preferred)
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub priority: Option<u32>,
|
||||
|
||||
/// Worker cost factor (optional, default: 1.0)
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub cost: Option<f32>,
|
||||
|
||||
/// Worker type (optional: "regular", "prefill", "decode")
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub worker_type: Option<String>,
|
||||
|
||||
/// Bootstrap port for prefill workers (optional)
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub bootstrap_port: Option<u16>,
|
||||
|
||||
/// Runtime type (optional: "sglang", "vllm", default: "sglang")
|
||||
/// Only relevant for gRPC workers
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub runtime: Option<String>,
|
||||
|
||||
// gRPC-specific configuration (optional, ignored in HTTP mode)
|
||||
/// Tokenizer path for gRPC mode
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub tokenizer_path: Option<String>,
|
||||
|
||||
/// Reasoning parser type for gRPC mode
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub reasoning_parser: Option<String>,
|
||||
|
||||
/// Tool parser type for gRPC mode
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub tool_parser: Option<String>,
|
||||
|
||||
/// Chat template for gRPC mode
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub chat_template: Option<String>,
|
||||
|
||||
/// Additional labels (optional)
|
||||
#[serde(default, skip_serializing_if = "HashMap::is_empty")]
|
||||
pub labels: HashMap<String, String>,
|
||||
|
||||
/// Health check timeout in seconds (default: 30)
|
||||
#[serde(default = "default_health_check_timeout")]
|
||||
pub health_check_timeout_secs: u64,
|
||||
|
||||
/// Health check interval in seconds (default: 60)
|
||||
#[serde(default = "default_health_check_interval")]
|
||||
pub health_check_interval_secs: u64,
|
||||
|
||||
/// Number of successful health checks needed to mark worker as healthy (default: 2)
|
||||
#[serde(default = "default_health_success_threshold")]
|
||||
pub health_success_threshold: u32,
|
||||
|
||||
/// Number of failed health checks before marking worker as unhealthy (default: 3)
|
||||
#[serde(default = "default_health_failure_threshold")]
|
||||
pub health_failure_threshold: u32,
|
||||
|
||||
/// Disable periodic health checks for this worker (default: false)
|
||||
#[serde(default)]
|
||||
pub disable_health_check: bool,
|
||||
|
||||
/// Maximum connection attempts during worker registration (default: 20)
|
||||
#[serde(default = "default_max_connection_attempts")]
|
||||
pub max_connection_attempts: u32,
|
||||
|
||||
/// Enable data parallelism aware scheduling (default: false)
|
||||
#[serde(default)]
|
||||
pub dp_aware: bool,
|
||||
}
|
||||
|
||||
// Default value functions for serde
|
||||
fn default_health_check_timeout() -> u64 {
|
||||
30
|
||||
}
|
||||
|
||||
fn default_health_check_interval() -> u64 {
|
||||
60
|
||||
}
|
||||
|
||||
fn default_health_success_threshold() -> u32 {
|
||||
2
|
||||
}
|
||||
|
||||
fn default_health_failure_threshold() -> u32 {
|
||||
3
|
||||
}
|
||||
|
||||
fn default_max_connection_attempts() -> u32 {
|
||||
20
|
||||
}
|
||||
|
||||
/// Worker information for API responses
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct WorkerInfo {
|
||||
/// Worker unique identifier
|
||||
pub id: String,
|
||||
|
||||
/// Worker URL
|
||||
pub url: String,
|
||||
|
||||
/// Model ID this worker serves
|
||||
pub model_id: String,
|
||||
|
||||
/// Worker priority
|
||||
pub priority: u32,
|
||||
|
||||
/// Worker cost factor
|
||||
pub cost: f32,
|
||||
|
||||
/// Worker type
|
||||
pub worker_type: String,
|
||||
|
||||
/// Whether the worker is healthy
|
||||
pub is_healthy: bool,
|
||||
|
||||
/// Current load on the worker
|
||||
pub load: usize,
|
||||
|
||||
/// Connection mode (http or grpc)
|
||||
pub connection_mode: String,
|
||||
|
||||
/// Runtime type (sglang or vllm, for gRPC workers)
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub runtime_type: Option<String>,
|
||||
|
||||
// gRPC-specific fields (None for HTTP workers)
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub tokenizer_path: Option<String>,
|
||||
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub reasoning_parser: Option<String>,
|
||||
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub tool_parser: Option<String>,
|
||||
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub chat_template: Option<String>,
|
||||
|
||||
/// Bootstrap port for prefill workers
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub bootstrap_port: Option<u16>,
|
||||
|
||||
/// Additional metadata
|
||||
#[serde(skip_serializing_if = "HashMap::is_empty")]
|
||||
pub metadata: HashMap<String, String>,
|
||||
|
||||
/// Whether health checks are disabled for this worker
|
||||
pub disable_health_check: bool,
|
||||
|
||||
/// Job status for async operations (if available)
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub job_status: Option<JobStatus>,
|
||||
}
|
||||
|
||||
impl WorkerInfo {
|
||||
/// Create a partial WorkerInfo for pending workers (not yet registered).
|
||||
/// Used when a worker ID maps to a URL but the worker is still being registered.
|
||||
pub fn pending(worker_id: &str, url: String, job_status: Option<JobStatus>) -> Self {
|
||||
Self {
|
||||
id: worker_id.to_string(),
|
||||
url,
|
||||
model_id: UNKNOWN_MODEL_ID.to_string(),
|
||||
priority: 0,
|
||||
cost: 1.0,
|
||||
worker_type: UNKNOWN_MODEL_ID.to_string(),
|
||||
is_healthy: false,
|
||||
load: 0,
|
||||
connection_mode: UNKNOWN_MODEL_ID.to_string(),
|
||||
runtime_type: None,
|
||||
tokenizer_path: None,
|
||||
reasoning_parser: None,
|
||||
tool_parser: None,
|
||||
chat_template: None,
|
||||
bootstrap_port: None,
|
||||
metadata: HashMap::new(),
|
||||
disable_health_check: false,
|
||||
job_status,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Job status for async control plane operations
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct JobStatus {
|
||||
pub job_type: String,
|
||||
pub worker_url: String,
|
||||
pub status: String,
|
||||
pub message: Option<String>,
|
||||
pub timestamp: u64,
|
||||
}
|
||||
|
||||
/// Worker list response
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct WorkerListResponse {
|
||||
/// List of workers
|
||||
pub workers: Vec<WorkerInfo>,
|
||||
|
||||
/// Total count
|
||||
pub total: usize,
|
||||
|
||||
/// Statistics
|
||||
pub stats: WorkerStats,
|
||||
}
|
||||
|
||||
/// Worker statistics
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct WorkerStats {
|
||||
pub total_workers: usize,
|
||||
pub healthy_workers: usize,
|
||||
pub total_models: usize,
|
||||
pub total_load: usize,
|
||||
pub by_type: WorkerTypeStats,
|
||||
}
|
||||
|
||||
/// Worker statistics by type
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct WorkerTypeStats {
|
||||
pub regular: usize,
|
||||
pub prefill: usize,
|
||||
pub decode: usize,
|
||||
}
|
||||
|
||||
/// Worker update request
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct WorkerUpdateRequest {
|
||||
/// Update priority
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub priority: Option<u32>,
|
||||
|
||||
/// Update cost
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub cost: Option<f32>,
|
||||
|
||||
/// Update labels
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub labels: Option<HashMap<String, String>>,
|
||||
|
||||
/// Update API key (for key rotation)
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub api_key: Option<String>,
|
||||
|
||||
/// Update health check timeout in seconds
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub health_check_timeout_secs: Option<u64>,
|
||||
|
||||
/// Update health check interval in seconds
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub health_check_interval_secs: Option<u64>,
|
||||
|
||||
/// Update health success threshold
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub health_success_threshold: Option<u32>,
|
||||
|
||||
/// Update health failure threshold
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub health_failure_threshold: Option<u32>,
|
||||
|
||||
/// Disable periodic health checks for this worker
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub disable_health_check: Option<bool>,
|
||||
}
|
||||
|
||||
/// Generic API response
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct WorkerApiResponse {
|
||||
pub success: bool,
|
||||
pub message: String,
|
||||
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub worker: Option<WorkerInfo>,
|
||||
}
|
||||
|
||||
/// Error response
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct WorkerErrorResponse {
|
||||
pub error: String,
|
||||
pub code: String,
|
||||
}
|
||||
|
||||
/// Server info response from /get_server_info endpoint
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
pub struct ServerInfo {
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub model_id: Option<String>,
|
||||
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub model_path: Option<String>,
|
||||
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub priority: Option<u32>,
|
||||
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub cost: Option<f32>,
|
||||
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub worker_type: Option<String>,
|
||||
|
||||
// gRPC-specific
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub tokenizer_path: Option<String>,
|
||||
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub reasoning_parser: Option<String>,
|
||||
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub tool_parser: Option<String>,
|
||||
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub chat_template: Option<String>,
|
||||
}
|
||||
|
||||
/// Result from flush cache operations across workers
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct FlushCacheResult {
|
||||
/// URLs of workers where cache flush succeeded
|
||||
pub successful: Vec<String>,
|
||||
/// URLs and error messages for workers where cache flush failed
|
||||
pub failed: Vec<(String, String)>,
|
||||
/// Total number of workers attempted
|
||||
pub total_workers: usize,
|
||||
/// Number of HTTP workers (gRPC workers don't support flush cache)
|
||||
pub http_workers: usize,
|
||||
/// Human-readable summary message
|
||||
pub message: String,
|
||||
}
|
||||
|
||||
/// Result from getting worker loads
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct WorkerLoadsResult {
|
||||
/// Worker URL and load pairs
|
||||
pub loads: Vec<WorkerLoadInfo>,
|
||||
/// Total number of workers
|
||||
pub total_workers: usize,
|
||||
/// Number of workers with successful load fetches
|
||||
pub successful: usize,
|
||||
/// Number of workers with failed load fetches
|
||||
pub failed: usize,
|
||||
}
|
||||
|
||||
/// Individual worker load information
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct WorkerLoadInfo {
|
||||
/// Worker URL
|
||||
pub worker: String,
|
||||
/// Worker type (regular, prefill, decode)
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub worker_type: Option<String>,
|
||||
/// Current load (-1 indicates failure to fetch)
|
||||
pub load: isize,
|
||||
}
|
||||
Reference in New Issue
Block a user