[model-gateway] reducing cpu overhead in various of places (#14658)
This commit is contained in:
Simo Lin
@@ -66,21 +66,67 @@ pub enum MessageContent {
|
||||
}
|
||||
|
||||
impl MessageContent {
|
||||
/// Returns the text content, cloning only when necessary.
|
||||
/// For simple text, returns a clone of the string.
|
||||
/// For parts, concatenates text parts with spaces.
|
||||
pub fn to_simple_string(&self) -> String {
|
||||
match self {
|
||||
MessageContent::Text(text) => text.clone(),
|
||||
MessageContent::Parts(parts) => {
|
||||
let texts: Vec<String> = parts
|
||||
// Pre-count text parts to avoid intermediate Vec allocation
|
||||
let text_parts: Vec<&str> = parts
|
||||
.iter()
|
||||
.filter_map(|part| match part {
|
||||
ContentPart::Text { text } => Some(text.clone()),
|
||||
ContentPart::Text { text } => Some(text.as_str()),
|
||||
_ => None,
|
||||
})
|
||||
.collect();
|
||||
texts.join(" ")
|
||||
text_parts.join(" ")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Appends text content directly to a buffer, avoiding intermediate allocations.
|
||||
/// Returns true if any content was appended.
|
||||
#[inline]
|
||||
pub fn append_text_to(&self, buffer: &mut String) -> bool {
|
||||
match self {
|
||||
MessageContent::Text(text) => {
|
||||
if !text.is_empty() {
|
||||
buffer.push_str(text);
|
||||
true
|
||||
} else {
|
||||
false
|
||||
}
|
||||
}
|
||||
MessageContent::Parts(parts) => {
|
||||
let mut appended = false;
|
||||
for part in parts {
|
||||
if let ContentPart::Text { text } = part {
|
||||
if !text.is_empty() {
|
||||
if appended {
|
||||
buffer.push(' ');
|
||||
}
|
||||
buffer.push_str(text);
|
||||
appended = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
appended
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns true if this content contains any non-empty text.
|
||||
#[inline]
|
||||
pub fn has_text(&self) -> bool {
|
||||
match self {
|
||||
MessageContent::Text(text) => !text.is_empty(),
|
||||
MessageContent::Parts(parts) => parts
|
||||
.iter()
|
||||
.any(|part| matches!(part, ContentPart::Text { text } if !text.is_empty())),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
@@ -581,33 +627,66 @@ impl GenerationRequest for ChatCompletionRequest {
|
||||
|
||||
fn extract_text_for_routing(&self) -> String {
|
||||
// Extract text from messages for routing decisions
|
||||
self.messages
|
||||
.iter()
|
||||
.filter_map(|msg| match msg {
|
||||
ChatMessage::System { content, .. } => Some(content.to_simple_string()),
|
||||
ChatMessage::User { content, .. } => Some(content.to_simple_string()),
|
||||
// Use a single buffer to avoid intermediate Vec<String> allocations
|
||||
let mut buffer = String::new();
|
||||
let mut has_content = false;
|
||||
|
||||
for msg in &self.messages {
|
||||
match msg {
|
||||
ChatMessage::System { content, .. } | ChatMessage::User { content, .. } => {
|
||||
if has_content && content.has_text() {
|
||||
buffer.push(' ');
|
||||
}
|
||||
if content.append_text_to(&mut buffer) {
|
||||
has_content = true;
|
||||
}
|
||||
}
|
||||
ChatMessage::Assistant {
|
||||
content,
|
||||
reasoning_content,
|
||||
..
|
||||
} => {
|
||||
// Combine content and reasoning content for routing decisions
|
||||
let main_content = content
|
||||
.as_ref()
|
||||
.map(|c| c.to_simple_string())
|
||||
.unwrap_or_default();
|
||||
let reasoning = reasoning_content.clone().unwrap_or_default();
|
||||
if main_content.is_empty() && reasoning.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(format!("{} {}", main_content, reasoning).trim().to_string())
|
||||
// Append main content
|
||||
if let Some(c) = content {
|
||||
if has_content && c.has_text() {
|
||||
buffer.push(' ');
|
||||
}
|
||||
if c.append_text_to(&mut buffer) {
|
||||
has_content = true;
|
||||
}
|
||||
}
|
||||
// Append reasoning content
|
||||
if let Some(reasoning) = reasoning_content {
|
||||
if !reasoning.is_empty() {
|
||||
if has_content {
|
||||
buffer.push(' ');
|
||||
}
|
||||
buffer.push_str(reasoning);
|
||||
has_content = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
ChatMessage::Tool { content, .. } => Some(content.to_simple_string()),
|
||||
ChatMessage::Function { content, .. } => Some(content.clone()),
|
||||
})
|
||||
.collect::<Vec<String>>()
|
||||
.join(" ")
|
||||
ChatMessage::Tool { content, .. } => {
|
||||
if has_content && content.has_text() {
|
||||
buffer.push(' ');
|
||||
}
|
||||
if content.append_text_to(&mut buffer) {
|
||||
has_content = true;
|
||||
}
|
||||
}
|
||||
ChatMessage::Function { content, .. } => {
|
||||
if !content.is_empty() {
|
||||
if has_content {
|
||||
buffer.push(' ');
|
||||
}
|
||||
buffer.push_str(content);
|
||||
has_content = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
buffer
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -65,15 +65,80 @@ impl StringOrArray {
|
||||
}
|
||||
}
|
||||
|
||||
/// Convert to a vector of strings
|
||||
/// Convert to a vector of strings (clones the data)
|
||||
pub fn to_vec(&self) -> Vec<String> {
|
||||
match self {
|
||||
StringOrArray::String(s) => vec![s.clone()],
|
||||
StringOrArray::Array(arr) => arr.clone(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns an iterator over string references without cloning.
|
||||
/// Use this instead of `to_vec()` when you only need to iterate.
|
||||
pub fn iter(&self) -> StringOrArrayIter<'_> {
|
||||
StringOrArrayIter {
|
||||
inner: self,
|
||||
index: 0,
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the first string, or None if empty
|
||||
pub fn first(&self) -> Option<&str> {
|
||||
match self {
|
||||
StringOrArray::String(s) => {
|
||||
if s.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(s)
|
||||
}
|
||||
}
|
||||
StringOrArray::Array(arr) => arr.first().map(|s| s.as_str()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Iterator over StringOrArray that yields string references without cloning
|
||||
pub struct StringOrArrayIter<'a> {
|
||||
inner: &'a StringOrArray,
|
||||
index: usize,
|
||||
}
|
||||
|
||||
impl<'a> Iterator for StringOrArrayIter<'a> {
|
||||
type Item = &'a str;
|
||||
|
||||
fn next(&mut self) -> Option<Self::Item> {
|
||||
match self.inner {
|
||||
StringOrArray::String(s) => {
|
||||
if self.index == 0 {
|
||||
self.index = 1;
|
||||
Some(s.as_str())
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
StringOrArray::Array(arr) => {
|
||||
if self.index < arr.len() {
|
||||
let item = &arr[self.index];
|
||||
self.index += 1;
|
||||
Some(item.as_str())
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn size_hint(&self) -> (usize, Option<usize>) {
|
||||
let remaining = match self.inner {
|
||||
StringOrArray::String(_) => 1 - self.index,
|
||||
StringOrArray::Array(arr) => arr.len() - self.index,
|
||||
};
|
||||
(remaining, Some(remaining))
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a> ExactSizeIterator for StringOrArrayIter<'a> {}
|
||||
|
||||
/// Validates stop sequences (max 4, non-empty strings)
|
||||
/// Used by both ChatCompletionRequest and ResponsesRequest
|
||||
pub fn validate_stop(stop: &StringOrArray) -> Result<(), validator::ValidationError> {
|
||||
|
||||
Reference in New Issue
Block a user