chenchenghao/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

Add code field and unify error responses for router (#15028)

Browse Source
This commit is contained in:
fzyzcjy
2025-12-13 13:21:34 +08:00
committed by GitHub
parent 05325db349
commit 9d3b411c56
26 changed files with 354 additions and 155 deletions
Download Patch File Download Diff File Expand all files Collapse all files
sgl-model-gateway/src/routers/error.rs
+103 -35
View File
@@ -5,88 +5,156 @@ use axum::{
};
use serde_json::json;
pub fn internal_error(message: impl Into<String>) -> Response {
create_error(StatusCode::INTERNAL_SERVER_ERROR, "internal_error", message)
pub fn internal_error(code: impl Into<String>, message: impl Into<String>) -> Response {
create_error(StatusCode::INTERNAL_SERVER_ERROR, code, message)
}
pub fn bad_request(message: impl Into<String>) -> Response {
create_error(StatusCode::BAD_REQUEST, "invalid_request_error", message)
pub fn bad_request(code: impl Into<String>, message: impl Into<String>) -> Response {
create_error(StatusCode::BAD_REQUEST, code, message)
}
pub fn not_found(message: impl Into<String>) -> Response {
create_error(StatusCode::NOT_FOUND, "invalid_request_error", message)
pub fn not_found(code: impl Into<String>, message: impl Into<String>) -> Response {
create_error(StatusCode::NOT_FOUND, code, message)
}
pub fn service_unavailable(message: impl Into<String>) -> Response {
create_error(
StatusCode::SERVICE_UNAVAILABLE,
"service_unavailable",
message,
)
pub fn service_unavailable(code: impl Into<String>, message: impl Into<String>) -> Response {
create_error(StatusCode::SERVICE_UNAVAILABLE, code, message)
}
pub fn failed_dependency(message: impl Into<String>) -> Response {
create_error(
StatusCode::FAILED_DEPENDENCY,
"external_connector_error",
message,
)
pub fn failed_dependency(code: impl Into<String>, message: impl Into<String>) -> Response {
create_error(StatusCode::FAILED_DEPENDENCY, code, message)
}
pub fn not_implemented(message: impl Into<String>) -> Response {
create_error(
StatusCode::NOT_IMPLEMENTED,
"not_implemented_error",
message,
)
pub fn not_implemented(code: impl Into<String>, message: impl Into<String>) -> Response {
create_error(StatusCode::NOT_IMPLEMENTED, code, message)
}
fn create_error(status_code: StatusCode, error_type: &str, message: impl Into<String>) -> Response {
let msg = message.into();
fn create_error(
status: StatusCode,
code: impl Into<String>,
message: impl Into<String>,
) -> Response {
(
status_code,
status,
Json(json!({
"error": {
"message": msg,
"type": error_type,
"code": status_code.as_u16()
"message": message.into(),
"type": status_code_to_str(status),
"code": code.into(),
}
})),
)
.into_response()
}
fn status_code_to_str(status_code: StatusCode) -> &'static str {
match status_code {
// 1xx
StatusCode::CONTINUE => "continue",
StatusCode::SWITCHING_PROTOCOLS => "switching_protocols",
StatusCode::PROCESSING => "processing",
StatusCode::EARLY_HINTS => "early_hints",
// 2xx
StatusCode::OK => "ok",
StatusCode::CREATED => "created",
StatusCode::ACCEPTED => "accepted",
StatusCode::NON_AUTHORITATIVE_INFORMATION => "non_authoritative_information",
StatusCode::NO_CONTENT => "no_content",
StatusCode::RESET_CONTENT => "reset_content",
StatusCode::PARTIAL_CONTENT => "partial_content",
StatusCode::MULTI_STATUS => "multi_status",
StatusCode::ALREADY_REPORTED => "already_reported",
StatusCode::IM_USED => "im_used",
// 3xx
StatusCode::MULTIPLE_CHOICES => "multiple_choices",
StatusCode::MOVED_PERMANENTLY => "moved_permanently",
StatusCode::FOUND => "found",
StatusCode::SEE_OTHER => "see_other",
StatusCode::NOT_MODIFIED => "not_modified",
StatusCode::USE_PROXY => "use_proxy",
StatusCode::TEMPORARY_REDIRECT => "temporary_redirect",
StatusCode::PERMANENT_REDIRECT => "permanent_redirect",
// 4xx
StatusCode::BAD_REQUEST => "bad_request",
StatusCode::UNAUTHORIZED => "unauthorized",
StatusCode::PAYMENT_REQUIRED => "payment_required",
StatusCode::FORBIDDEN => "forbidden",
StatusCode::NOT_FOUND => "not_found",
StatusCode::METHOD_NOT_ALLOWED => "method_not_allowed",
StatusCode::NOT_ACCEPTABLE => "not_acceptable",
StatusCode::PROXY_AUTHENTICATION_REQUIRED => "proxy_authentication_required",
StatusCode::REQUEST_TIMEOUT => "request_timeout",
StatusCode::CONFLICT => "conflict",
StatusCode::GONE => "gone",
StatusCode::LENGTH_REQUIRED => "length_required",
StatusCode::PRECONDITION_FAILED => "precondition_failed",
StatusCode::PAYLOAD_TOO_LARGE => "payload_too_large",
StatusCode::URI_TOO_LONG => "uri_too_long",
StatusCode::UNSUPPORTED_MEDIA_TYPE => "unsupported_media_type",
StatusCode::RANGE_NOT_SATISFIABLE => "range_not_satisfiable",
StatusCode::EXPECTATION_FAILED => "expectation_failed",
StatusCode::IM_A_TEAPOT => "im_a_teapot",
StatusCode::MISDIRECTED_REQUEST => "misdirected_request",
StatusCode::UNPROCESSABLE_ENTITY => "unprocessable_entity",
StatusCode::LOCKED => "locked",
StatusCode::FAILED_DEPENDENCY => "failed_dependency",
StatusCode::UPGRADE_REQUIRED => "upgrade_required",
StatusCode::PRECONDITION_REQUIRED => "precondition_required",
StatusCode::TOO_MANY_REQUESTS => "too_many_requests",
StatusCode::REQUEST_HEADER_FIELDS_TOO_LARGE => "request_header_fields_too_large",
StatusCode::UNAVAILABLE_FOR_LEGAL_REASONS => "unavailable_for_legal_reasons",
// 5xx
StatusCode::INTERNAL_SERVER_ERROR => "internal_server_error",
StatusCode::NOT_IMPLEMENTED => "not_implemented",
StatusCode::BAD_GATEWAY => "bad_gateway",
StatusCode::SERVICE_UNAVAILABLE => "service_unavailable",
StatusCode::GATEWAY_TIMEOUT => "gateway_timeout",
StatusCode::HTTP_VERSION_NOT_SUPPORTED => "http_version_not_supported",
StatusCode::VARIANT_ALSO_NEGOTIATES => "variant_also_negotiates",
StatusCode::INSUFFICIENT_STORAGE => "insufficient_storage",
StatusCode::LOOP_DETECTED => "loop_detected",
StatusCode::NOT_EXTENDED => "not_extended",
StatusCode::NETWORK_AUTHENTICATION_REQUIRED => "network_authentication_required",
_ => "unknown_status_code",
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_internal_error_string() {
let response = internal_error("Test error");
let response = internal_error("test_error", "Test error");
assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR);
}
#[test]
fn test_internal_error_format() {
let response = internal_error(format!("Error: {}", 42));
let response = internal_error("test_error", format!("Error: {}", 42));
assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR);
}
#[test]
fn test_bad_request() {
let response = bad_request("Invalid input");
let response = bad_request("invalid_input", "Invalid input");
assert_eq!(response.status(), StatusCode::BAD_REQUEST);
}
#[test]
fn test_not_found() {
let response = not_found("Resource not found");
let response = not_found("resource_not_found", "Resource not found");
assert_eq!(response.status(), StatusCode::NOT_FOUND);
}
#[test]
fn test_service_unavailable() {
let response = service_unavailable("No workers");
let response = service_unavailable("no_workers", "No workers");
assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
}
}
sgl-model-gateway/src/routers/grpc/common/response_collection.rs
+4 -1
View File
@@ -58,7 +58,10 @@ pub async fn collect_responses(
};
if all_responses.is_empty() {
return Err(error::internal_error("No responses from server"));
return Err(error::internal_error(
"no_responses_from_server",
"No responses from server",
));
}
Ok(all_responses)
sgl-model-gateway/src/routers/grpc/common/responses/utils.rs
+1
View File
@@ -49,6 +49,7 @@ pub async fn ensure_mcp_connection(
"Failed to connect to MCP server"
);
return Err(error::failed_dependency(
"connect_mcp_server_failed",
"Failed to connect to MCP server. Check server_url and authorization.",
));
}
sgl-model-gateway/src/routers/grpc/common/stages/client_acquisition.rs
+5 -1
View File
@@ -24,7 +24,10 @@ impl PipelineStage for ClientAcquisitionStage {
function = "ClientAcquisitionStage::execute",
"Worker selection stage not completed"
);
error::internal_error("Worker selection not completed")
error::internal_error(
"worker_selection_not_completed",
"Worker selection not completed",
)
})?;
let clients = match workers {
@@ -43,6 +46,7 @@ impl PipelineStage for ClientAcquisitionStage {
"vLLM backend does not support dual (PD disaggregated) mode"
);
return Err(error::bad_request(
"vllm_pd_mode_not_supported",
"vLLM backend does not support prefill/decode disaggregated mode. \
Please use runtime_type: sglang for PD mode, or use a regular (non-PD) worker configuration."
));
sgl-model-gateway/src/routers/grpc/common/stages/dispatch_metadata.rs
+1 -1
View File
@@ -23,7 +23,7 @@ impl PipelineStage for DispatchMetadataStage {
function = "DispatchMetadataStage::execute",
"Proto request not built"
);
error::internal_error("Proto request not built")
error::internal_error("proto_request_not_built", "Proto request not built")
})?;
let request_id = proto_request.request_id().to_string();
sgl-model-gateway/src/routers/grpc/common/stages/request_execution.rs
+25 -13
View File
@@ -42,7 +42,7 @@ impl PipelineStage for RequestExecutionStage {
function = "RequestExecutionStage::execute",
"Proto request not built"
);
error::internal_error("Proto request not built")
error::internal_error("proto_request_not_built", "Proto request not built")
})?;
let clients = ctx.state.clients.as_mut().ok_or_else(|| {
@@ -50,7 +50,10 @@ impl PipelineStage for RequestExecutionStage {
function = "RequestExecutionStage::execute",
"Client acquisition not completed"
);
error::internal_error("Client acquisition not completed")
error::internal_error(
"client_acquisition_not_completed",
"Client acquisition not completed",
)
})?;
// Extract dispatch metadata for tracing span
@@ -108,7 +111,10 @@ impl RequestExecutionStage {
function = "execute_single",
"Expected single client but got dual"
);
error::internal_error("Expected single client but got dual")
error::internal_error(
"expected_single_client_got_dual",
"Expected single client but got dual",
)
})?;
let stream = client.generate(proto_request).await.map_err(|e| {
@@ -117,7 +123,10 @@ impl RequestExecutionStage {
error = %e,
"Failed to start generation"
);
error::internal_error(format!("Failed to start generation: {}", e))
error::internal_error(
"start_generation_failed",
format!("Failed to start generation: {}", e),
)
})?;
Ok(ExecutionResult::Single { stream })
@@ -133,7 +142,10 @@ impl RequestExecutionStage {
function = "execute_dual_dispatch",
"Expected dual clients but got single"
);
error::internal_error("Expected dual clients but got single")
error::internal_error(
"expected_dual_clients_got_single",
"Expected dual clients but got single",
)
})?;
let prefill_request = proto_request.clone_inner();
@@ -153,10 +165,10 @@ impl RequestExecutionStage {
error = %e,
"Prefill worker failed to start"
);
return Err(error::internal_error(format!(
"Prefill worker failed to start: {}",
e
)));
return Err(error::internal_error(
"prefill_worker_failed_to_start",
format!("Prefill worker failed to start: {}", e),
));
}
};
@@ -169,10 +181,10 @@ impl RequestExecutionStage {
error = %e,
"Decode worker failed to start"
);
return Err(error::internal_error(format!(
"Decode worker failed to start: {}",
e
)));
return Err(error::internal_error(
"decode_worker_failed_to_start",
format!("Decode worker failed to start: {}", e),
));
}
};
sgl-model-gateway/src/routers/grpc/common/stages/worker_selection.rs
+15 -9
View File
@@ -52,7 +52,10 @@ impl PipelineStage for WorkerSelectionStage {
function = "WorkerSelectionStage::execute",
"Preparation stage not completed"
);
error::internal_error("Preparation stage not completed")
error::internal_error(
"preparation_stage_not_completed",
"Preparation stage not completed",
)
})?;
// For Harmony, use selection_text produced during Harmony encoding
@@ -74,10 +77,10 @@ impl PipelineStage for WorkerSelectionStage {
model_id = ?ctx.input.model_id,
"No available workers for model"
);
return Err(error::service_unavailable(format!(
"No available workers for model: {:?}",
ctx.input.model_id
)));
return Err(error::service_unavailable(
"no_available_workers",
format!("No available workers for model: {:?}", ctx.input.model_id),
));
}
}
}
@@ -91,10 +94,13 @@ impl PipelineStage for WorkerSelectionStage {
model_id = ?ctx.input.model_id,
"No available PD worker pairs for model"
);
return Err(error::service_unavailable(format!(
"No available PD worker pairs for model: {:?}",
ctx.input.model_id
)));
return Err(error::service_unavailable(
"no_available_pd_worker_pairs",
format!(
"No available PD worker pairs for model: {:?}",
ctx.input.model_id
),
));
}
}
}
sgl-model-gateway/src/routers/grpc/harmony/processor.rs
+25 -7
View File
@@ -47,7 +47,10 @@ impl HarmonyResponseProcessor {
// Collect all completed responses (one per choice)
let all_responses = response_collection::collect_responses(execution_result, false).await?;
if all_responses.is_empty() {
return Err(error::internal_error("No responses from server"));
return Err(error::internal_error(
"no_responses_from_server",
"No responses from server",
));
}
// Build choices by parsing output with HarmonyParserAdapter
@@ -72,7 +75,10 @@ impl HarmonyResponseProcessor {
error = %e,
"Failed to create Harmony parser"
);
error::internal_error(format!("Failed to create Harmony parser: {}", e))
error::internal_error(
"create_harmony_parser_failed",
format!("Failed to create Harmony parser: {}", e),
)
})?;
// Parse Harmony channels with finish_reason and matched_stop
@@ -88,7 +94,10 @@ impl HarmonyResponseProcessor {
error = %e,
"Harmony parsing failed on complete response"
);
error::internal_error(format!("Harmony parsing failed: {}", e))
error::internal_error(
"harmony_parsing_failed",
format!("Harmony parsing failed: {}", e),
)
})?;
// Build response message (assistant)
@@ -187,13 +196,16 @@ impl HarmonyResponseProcessor {
// Collect all completed responses
let all_responses = response_collection::collect_responses(execution_result, false).await?;
if all_responses.is_empty() {
return Err(error::internal_error("No responses from server"));
return Err(error::internal_error(
"no_responses_from_server",
"No responses from server",
));
}
// For Responses API, we only process the first response (n=1)
let complete = all_responses
.first()
.ok_or_else(|| error::internal_error("No complete response"))?;
.ok_or_else(|| error::internal_error("no_complete_response", "No complete response"))?;
// Parse Harmony channels
let mut parser = HarmonyParserAdapter::new().map_err(|e| {
@@ -202,7 +214,10 @@ impl HarmonyResponseProcessor {
error = %e,
"Failed to create Harmony parser"
);
error::internal_error(format!("Failed to create Harmony parser: {}", e))
error::internal_error(
"create_harmony_parser_failed",
format!("Failed to create Harmony parser: {}", e),
)
})?;
// Convert matched_stop from proto to JSON
@@ -227,7 +242,10 @@ impl HarmonyResponseProcessor {
error = %e,
"Harmony parsing failed on complete response"
);
error::internal_error(format!("Harmony parsing failed: {}", e))
error::internal_error(
"harmony_parsing_failed",
format!("Harmony parsing failed: {}", e),
)
})?;
// VALIDATION: Check if model incorrectly generated Tool role messages
sgl-model-gateway/src/routers/grpc/harmony/responses.rs
+18 -12
View File
@@ -332,10 +332,10 @@ async fn execute_with_mcp_loop(
max_iterations = MAX_TOOL_ITERATIONS,
"Maximum tool iterations exceeded"
);
return Err(error::internal_error(format!(
"Maximum tool iterations ({}) exceeded",
MAX_TOOL_ITERATIONS
)));
return Err(error::internal_error(
"tool_iterations_exceeded",
format!("Maximum tool iterations ({}) exceeded", MAX_TOOL_ITERATIONS),
));
}
debug!(
@@ -1176,10 +1176,13 @@ async fn execute_mcp_tools(
error = %e,
"Failed to parse tool arguments JSON"
);
error::internal_error(format!(
"Invalid tool arguments JSON for tool '{}': {}",
tool_call.function.name, e
))
error::internal_error(
"invalid_tool_args",
format!(
"Invalid tool arguments JSON for tool '{}': {}",
tool_call.function.name, e
),
)
})?;
// Execute tool via MCP manager
@@ -1544,10 +1547,13 @@ async fn load_previous_messages(
error = %e,
"Failed to load previous response chain from storage"
);
error::internal_error(format!(
"Failed to load previous response chain for {}: {}",
prev_id_str, e
))
error::internal_error(
"load_previous_response_chain_failed",
format!(
"Failed to load previous response chain for {}: {}",
prev_id_str, e
),
)
})?;
// Build conversation history from stored responses
sgl-model-gateway/src/routers/grpc/harmony/stages/preparation.rs
+21 -12
View File
@@ -65,6 +65,7 @@ impl PipelineStage for HarmonyPreparationStage {
"Unsupported request type for Harmony pipeline"
);
return Err(error::bad_request(
"harmony_request_type_invalid",
"Only Chat and Responses requests supported in Harmony pipeline".to_string(),
));
}
@@ -91,6 +92,7 @@ impl HarmonyPreparationStage {
"logprobs requested but not supported for Harmony models"
);
return Err(error::bad_request(
"harmony_logprobs_not_supported",
"logprobs are not supported for Harmony models".to_string(),
));
}
@@ -112,7 +114,10 @@ impl HarmonyPreparationStage {
error = %e,
"Harmony build failed for chat request"
);
error::bad_request(format!("Harmony build failed: {}", e))
error::bad_request(
"harmony_build_failed",
format!("Harmony build failed: {}", e),
)
})?;
// Step 4: Store results
@@ -175,6 +180,7 @@ impl HarmonyPreparationStage {
"Conflicting constraints: both tool_choice and text format specified"
);
return Err(error::bad_request(
"conflicting_constraints",
"Cannot use both tool_choice (required/function) and text format (json_object/json_schema) simultaneously".to_string(),
));
}
@@ -188,7 +194,10 @@ impl HarmonyPreparationStage {
error = %e,
"Harmony build failed for responses request"
);
error::bad_request(format!("Harmony build failed: {}", e))
error::bad_request(
"harmony_build_failed",
format!("Harmony build failed: {}", e),
)
})?;
// Step 4: Store results with constraint
@@ -230,7 +239,7 @@ impl HarmonyPreparationStage {
error = %e,
"Failed to build text format structural tag for JsonObject"
);
Box::new(error::internal_error(e))
Box::new(error::internal_error("build_text_format_tag_failed", e))
})?;
Ok(Some(("structural_tag".to_string(), tag)))
}
@@ -241,7 +250,7 @@ impl HarmonyPreparationStage {
error = %e,
"Failed to build text format structural tag for JsonSchema"
);
Box::new(error::internal_error(e))
Box::new(error::internal_error("build_text_format_tag_failed", e))
})?;
Ok(Some(("structural_tag".to_string(), tag)))
}
@@ -310,10 +319,10 @@ impl HarmonyPreparationStage {
tool_name = %tool_name,
"Specified tool not found in tools list"
);
return Err(Box::new(error::bad_request(format!(
"Tool '{}' not found in tools list",
tool_name
))));
return Err(Box::new(error::bad_request(
"tool_not_found",
format!("Tool '{}' not found in tools list", tool_name),
)));
}
_ => {}
}
@@ -362,10 +371,10 @@ impl HarmonyPreparationStage {
error = %e,
"Failed to serialize structural tag"
);
Box::new(error::internal_error(format!(
"Failed to serialize structural tag: {}",
e
)))
Box::new(error::internal_error(
"serialize_structural_tag_failed",
format!("Failed to serialize structural tag: {}", e),
))
})
}
}
sgl-model-gateway/src/routers/grpc/harmony/stages/request_building.rs
+15 -4
View File
@@ -38,7 +38,7 @@ impl PipelineStage for HarmonyRequestBuildingStage {
function = "HarmonyRequestBuildingStage::execute",
"Preparation stage not completed"
);
error::internal_error("Preparation not completed")
error::internal_error("preparation_not_completed", "Preparation not completed")
})?;
// Get clients
@@ -47,7 +47,10 @@ impl PipelineStage for HarmonyRequestBuildingStage {
function = "HarmonyRequestBuildingStage::execute",
"Client acquisition stage not completed"
);
error::internal_error("Client acquisition not completed")
error::internal_error(
"client_acquisition_not_completed",
"Client acquisition not completed",
)
})?;
let builder_client = match clients {
ClientSelection::Single { client } => client,
@@ -57,6 +60,7 @@ impl PipelineStage for HarmonyRequestBuildingStage {
// Harmony model support not yet implemented for vLLM
if builder_client.is_vllm() {
return Err(error::not_implemented(
"harmony_vllm_not_supported",
"Harmony model support is not yet implemented for vLLM backend. \
Please use runtime_type: sglang for Harmony models.",
));
@@ -72,6 +76,7 @@ impl PipelineStage for HarmonyRequestBuildingStage {
"Generate request type not supported for Harmony models"
);
return Err(error::bad_request(
"harmony_generate_not_supported",
"Generate requests are not supported with Harmony models".to_string(),
));
}
@@ -102,7 +107,10 @@ impl PipelineStage for HarmonyRequestBuildingStage {
error = %e,
"Failed to build generate request from chat"
);
error::bad_request(format!("Invalid request parameters: {}", e))
error::bad_request(
"invalid_request_parameters",
format!("Invalid request parameters: {}", e),
)
})?
}
RequestType::Responses(request) => sglang_client
@@ -120,7 +128,10 @@ impl PipelineStage for HarmonyRequestBuildingStage {
error = %e,
"Failed to build generate request from responses"
);
error::bad_request(format!("Invalid request parameters: {}", e))
error::bad_request(
"invalid_request_parameters",
format!("Invalid request parameters: {}", e),
)
})?,
_ => unreachable!(),
};
sgl-model-gateway/src/routers/grpc/harmony/stages/response_processing.rs
+5 -4
View File
@@ -56,7 +56,7 @@ impl PipelineStage for HarmonyResponseProcessingStage {
request_type = "Chat",
"No execution result available"
);
error::internal_error("No execution result")
error::internal_error("no_execution_result", "No execution result")
})?;
let dispatch = ctx.state.dispatch.as_ref().cloned().ok_or_else(|| {
@@ -65,7 +65,7 @@ impl PipelineStage for HarmonyResponseProcessingStage {
request_type = "Chat",
"Dispatch metadata not set"
);
error::internal_error("Dispatch metadata not set")
error::internal_error("dispatch_metadata_not_set", "Dispatch metadata not set")
})?;
// For streaming, delegate to streaming processor and return SSE response
@@ -107,7 +107,7 @@ impl PipelineStage for HarmonyResponseProcessingStage {
request_type = "Responses",
"No execution result available"
);
error::internal_error("No execution result")
error::internal_error("no_execution_result", "No execution result")
})?;
let dispatch = ctx.state.dispatch.as_ref().cloned().ok_or_else(|| {
@@ -116,7 +116,7 @@ impl PipelineStage for HarmonyResponseProcessingStage {
request_type = "Responses",
"Dispatch metadata not set"
);
error::internal_error("Dispatch metadata not set")
error::internal_error("dispatch_metadata_not_set", "Dispatch metadata not set")
})?;
let responses_request = ctx.responses_request_arc();
@@ -134,6 +134,7 @@ impl PipelineStage for HarmonyResponseProcessingStage {
"Generate request type not supported in Harmony pipeline"
);
Err(error::internal_error(
"generate_requests_not_supported_in_harmony",
"Generate requests not supported in Harmony pipeline",
))
}
sgl-model-gateway/src/routers/grpc/pipeline.rs
+21 -8
View File
@@ -224,14 +224,14 @@ impl RequestPipeline {
function = "execute_chat",
"Wrong response type: expected Chat, got Generate"
);
error::internal_error("Internal error: wrong response type")
error::internal_error("wrong_response_type", "Internal error: wrong response type")
}
None => {
error!(
function = "execute_chat",
"No response produced by pipeline"
);
error::internal_error("No response produced")
error::internal_error("no_response_produced", "No response produced")
}
}
}
@@ -275,14 +275,14 @@ impl RequestPipeline {
function = "execute_generate",
"Wrong response type: expected Generate, got Chat"
);
error::internal_error("Internal error: wrong response type")
error::internal_error("wrong_response_type", "Internal error: wrong response type")
}
None => {
error!(
function = "execute_generate",
"No response produced by pipeline"
);
error::internal_error("No response produced")
error::internal_error("no_response_produced", "No response produced")
}
}
}
@@ -311,6 +311,7 @@ impl RequestPipeline {
"Streaming attempted in responses context"
);
return Err(error::bad_request(
"streaming_not_supported",
"Streaming is not supported in this context".to_string(),
));
}
@@ -337,14 +338,20 @@ impl RequestPipeline {
function = "execute_chat_for_responses",
"Wrong response type: expected Chat, got Generate"
);
Err(error::internal_error("Internal error: wrong response type"))
Err(error::internal_error(
"wrong_response_type",
"Internal error: wrong response type",
))
}
None => {
error!(
function = "execute_chat_for_responses",
"No response produced by pipeline"
);
Err(error::internal_error("No response produced"))
Err(error::internal_error(
"no_response_produced",
"No response produced",
))
}
}
}
@@ -415,7 +422,10 @@ impl RequestPipeline {
function = "execute_harmony_responses",
"No ResponsesIterationResult produced by pipeline"
);
error::internal_error("No ResponsesIterationResult produced by pipeline")
error::internal_error(
"no_responses_iteration_result",
"No ResponsesIterationResult produced by pipeline",
)
})
}
@@ -465,7 +475,10 @@ impl RequestPipeline {
function = "execute_harmony_responses_streaming",
"No ExecutionResult produced by pipeline"
);
error::internal_error("No ExecutionResult produced by pipeline")
error::internal_error(
"no_execution_result_produced",
"No ExecutionResult produced by pipeline",
)
})
}
}
sgl-model-gateway/src/routers/grpc/regular/processor.rs
+8 -8
View File
@@ -278,10 +278,10 @@ impl ResponseProcessor {
{
Ok(choice) => choices.push(choice),
Err(e) => {
return Err(error::internal_error(format!(
"Failed to process choice {}: {}",
index, e
)));
return Err(error::internal_error(
"process_choice_failed",
format!("Failed to process choice {}: {}", index, e),
));
}
}
}
@@ -380,10 +380,10 @@ impl ResponseProcessor {
let outputs = match stop_decoder.process_tokens(complete.output_ids()) {
Ok(outputs) => outputs,
Err(e) => {
return Err(error::internal_error(format!(
"Failed to process tokens: {}",
e
)))
return Err(error::internal_error(
"process_tokens_failed",
format!("Failed to process tokens: {}", e),
))
}
};
sgl-model-gateway/src/routers/grpc/regular/responses/handlers.rs
+19 -7
View File
@@ -607,7 +607,10 @@ async fn execute_without_mcp(
error = %e,
"Failed to convert ResponsesRequest to ChatCompletionRequest"
);
error::bad_request(format!("Failed to convert request: {}", e))
error::bad_request(
"convert_request_failed",
format!("Failed to convert request: {}", e),
)
})?;
// Execute chat pipeline (errors already have proper HTTP status codes)
@@ -628,7 +631,10 @@ async fn execute_without_mcp(
error = %e,
"Failed to convert ChatCompletionResponse to ResponsesResponse"
);
error::internal_error(format!("Failed to convert to responses format: {}", e))
error::internal_error(
"convert_to_responses_format_failed",
format!("Failed to convert to responses format: {}", e),
)
})
}
@@ -713,14 +719,20 @@ async fn load_conversation_history(
error = %e,
"Failed to check conversation existence in storage"
);
error::internal_error(format!("Failed to check conversation: {}", e))
error::internal_error(
"check_conversation_failed",
format!("Failed to check conversation: {}", e),
)
})?;
if conversation.is_none() {
return Err(error::not_found(format!(
"Conversation '{}' not found. Please create the conversation first using the conversations API.",
conv_id_str
)));
return Err(error::not_found(
"conversation_not_found",
format!(
"Conversation '{}' not found. Please create the conversation first using the conversations API.",
conv_id_str
)
));
}
// Load conversation history
sgl-model-gateway/src/routers/grpc/regular/responses/tool_loop.rs
+16 -4
View File
@@ -257,7 +257,10 @@ pub(super) async fn execute_tool_loop(
error = %e,
"Failed to convert ResponsesRequest to ChatCompletionRequest in tool loop"
);
error::bad_request(format!("Failed to convert request: {}", e))
error::bad_request(
"convert_request_failed",
format!("Failed to convert request: {}", e),
)
})?;
// Prepare tools and tool_choice for this iteration
@@ -315,7 +318,10 @@ pub(super) async fn execute_tool_loop(
context = "function_tool_calls",
"Failed to convert ChatCompletionResponse to ResponsesResponse"
);
error::internal_error(format!("Failed to convert to responses format: {}", e))
error::internal_error(
"convert_to_responses_format_failed",
format!("Failed to convert to responses format: {}", e),
)
})?;
// Return response with function tool calls to caller
@@ -352,7 +358,10 @@ pub(super) async fn execute_tool_loop(
context = "max_tool_calls_limit",
"Failed to convert ChatCompletionResponse to ResponsesResponse"
);
error::internal_error(format!("Failed to convert to responses format: {}", e))
error::internal_error(
"convert_to_responses_format_failed",
format!("Failed to convert to responses format: {}", e),
)
})?;
// Mark as completed but with incomplete details
@@ -481,7 +490,10 @@ pub(super) async fn execute_tool_loop(
context = "final_response",
"Failed to convert ChatCompletionResponse to ResponsesResponse"
);
error::internal_error(format!("Failed to convert to responses format: {}", e))
error::internal_error(
"convert_to_responses_format_failed",
format!("Failed to convert to responses format: {}", e),
)
})?;
// Inject MCP metadata into output
sgl-model-gateway/src/routers/grpc/regular/stages/chat/preparation.rs
+6 -3
View File
@@ -54,7 +54,7 @@ impl ChatPreparationStage {
Ok(msgs) => msgs,
Err(e) => {
error!(function = "ChatPreparationStage::execute", error = %e, "Failed to process chat messages");
return Err(error::bad_request(e));
return Err(error::bad_request("process_messages_failed", e));
}
};
@@ -63,7 +63,10 @@ impl ChatPreparationStage {
Ok(encoding) => encoding,
Err(e) => {
error!(function = "ChatPreparationStage::execute", error = %e, "Tokenization failed");
return Err(error::internal_error(format!("Tokenization failed: {}", e)));
return Err(error::internal_error(
"tokenization_failed",
format!("Tokenization failed: {}", e),
));
}
};
@@ -74,7 +77,7 @@ impl ChatPreparationStage {
utils::generate_tool_constraints(tools, &request.tool_choice, &request.model)
.map_err(|e| {
error!(function = "ChatPreparationStage::execute", error = %e, "Invalid tool configuration");
error::bad_request(format!("Invalid tool configuration: {}", e))
error::bad_request("invalid_tool_configuration", format!("Invalid tool configuration: {}", e))
})?
} else {
None
sgl-model-gateway/src/routers/grpc/regular/stages/chat/request_building.rs
+7 -4
View File
@@ -36,7 +36,7 @@ impl PipelineStage for ChatRequestBuildingStage {
function = "ChatRequestBuildingStage::execute",
"Preparation not completed"
);
error::internal_error("Preparation not completed")
error::internal_error("preparation_not_completed", "Preparation not completed")
})?;
let clients = ctx.state.clients.as_ref().ok_or_else(|| {
@@ -44,7 +44,10 @@ impl PipelineStage for ChatRequestBuildingStage {
function = "ChatRequestBuildingStage::execute",
"Client acquisition not completed"
);
error::internal_error("Client acquisition not completed")
error::internal_error(
"client_acquisition_not_completed",
"Client acquisition not completed",
)
})?;
let chat_request = ctx.chat_request_arc();
@@ -77,7 +80,7 @@ impl PipelineStage for ChatRequestBuildingStage {
)
.map_err(|e| {
error!(function = "ChatRequestBuildingStage::execute", error = %e, "Failed to build SGLang generate request");
error::bad_request(format!("Invalid request parameters: {}", e))
error::bad_request("invalid_request_parameters", format!("Invalid request parameters: {}", e))
})?;
ProtoGenerateRequest::Sglang(Box::new(req))
}
@@ -92,7 +95,7 @@ impl PipelineStage for ChatRequestBuildingStage {
)
.map_err(|e| {
error!(function = "ChatRequestBuildingStage::execute", error = %e, "Failed to build vLLM generate request");
error::bad_request(format!("Invalid request parameters: {}", e))
error::bad_request("invalid_request_parameters", format!("Invalid request parameters: {}", e))
})?;
ProtoGenerateRequest::Vllm(Box::new(req))
}
sgl-model-gateway/src/routers/grpc/regular/stages/chat/response_processing.rs
+6 -3
View File
@@ -62,7 +62,7 @@ impl ChatResponseProcessingStage {
function = "ChatResponseProcessingStage::execute",
"No execution result"
);
error::internal_error("No execution result")
error::internal_error("no_execution_result", "No execution result")
})?;
// Get dispatch metadata (needed by both streaming and non-streaming)
@@ -75,7 +75,7 @@ impl ChatResponseProcessingStage {
function = "ChatResponseProcessingStage::execute",
"Dispatch metadata not set"
);
error::internal_error("Dispatch metadata not set")
error::internal_error("dispatch_metadata_not_set", "Dispatch metadata not set")
})?
.clone();
@@ -100,7 +100,10 @@ impl ChatResponseProcessingStage {
function = "ChatResponseProcessingStage::execute",
"Stop decoder not initialized"
);
error::internal_error("Stop decoder not initialized")
error::internal_error(
"stop_decoder_not_initialized",
"Stop decoder not initialized",
)
})?;
let response = self
sgl-model-gateway/src/routers/grpc/regular/stages/generate/preparation.rs
+1 -1
View File
@@ -49,7 +49,7 @@ impl GeneratePreparationStage {
Ok(res) => res,
Err(msg) => {
error!(function = "GeneratePreparationStage::execute", error = %msg, "Failed to resolve generate input");
return Err(error::bad_request(msg));
return Err(error::bad_request("resolve_input_failed", msg));
}
};
sgl-model-gateway/src/routers/grpc/regular/stages/generate/request_building.rs
+7 -4
View File
@@ -36,7 +36,7 @@ impl PipelineStage for GenerateRequestBuildingStage {
function = "GenerateRequestBuildingStage::execute",
"Preparation not completed"
);
error::internal_error("Preparation not completed")
error::internal_error("preparation_not_completed", "Preparation not completed")
})?;
let clients = ctx.state.clients.as_ref().ok_or_else(|| {
@@ -44,7 +44,10 @@ impl PipelineStage for GenerateRequestBuildingStage {
function = "GenerateRequestBuildingStage::execute",
"Client acquisition not completed"
);
error::internal_error("Client acquisition not completed")
error::internal_error(
"client_acquisition_not_completed",
"Client acquisition not completed",
)
})?;
let generate_request = ctx.generate_request_arc();
@@ -73,7 +76,7 @@ impl PipelineStage for GenerateRequestBuildingStage {
)
.map_err(|e| {
error!(function = "GenerateRequestBuildingStage::execute", error = %e, "Failed to build SGLang generate request");
error::bad_request(e)
error::bad_request("build_request_failed", e)
})?;
ProtoGenerateRequest::Sglang(Box::new(req))
}
@@ -87,7 +90,7 @@ impl PipelineStage for GenerateRequestBuildingStage {
)
.map_err(|e| {
error!(function = "GenerateRequestBuildingStage::execute", error = %e, "Failed to build vLLM generate request");
error::bad_request(e)
error::bad_request("build_request_failed", e)
})?;
ProtoGenerateRequest::Vllm(Box::new(req))
}
sgl-model-gateway/src/routers/grpc/regular/stages/generate/response_processing.rs
+6 -3
View File
@@ -60,7 +60,7 @@ impl GenerateResponseProcessingStage {
function = "GenerateResponseProcessingStage::execute",
"No execution result"
);
error::internal_error("No execution result")
error::internal_error("no_execution_result", "No execution result")
})?;
// Get dispatch metadata (needed by both streaming and non-streaming)
@@ -73,7 +73,7 @@ impl GenerateResponseProcessingStage {
function = "GenerateResponseProcessingStage::execute",
"Dispatch metadata not set"
);
error::internal_error("Dispatch metadata not set")
error::internal_error("dispatch_metadata_not_set", "Dispatch metadata not set")
})?
.clone();
@@ -97,7 +97,10 @@ impl GenerateResponseProcessingStage {
function = "GenerateResponseProcessingStage::execute",
"Stop decoder not initialized"
);
error::internal_error("Stop decoder not initialized")
error::internal_error(
"stop_decoder_not_initialized",
"Stop decoder not initialized",
)
})?;
let result_array = self
sgl-model-gateway/src/routers/grpc/regular/stages/preparation.rs
+1
View File
@@ -49,6 +49,7 @@ impl PipelineStage for PreparationStage {
"RequestType::Responses reached regular preparation stage"
);
Err(grpc_error::internal_error(
"responses_in_wrong_pipeline",
"RequestType::Responses reached regular preparation stage",
))
}
sgl-model-gateway/src/routers/grpc/regular/stages/request_building.rs
+1
View File
@@ -40,6 +40,7 @@ impl PipelineStage for RequestBuildingStage {
"RequestType::Responses reached regular request building stage"
);
Err(grpc_error::internal_error(
"responses_in_wrong_pipeline",
"RequestType::Responses reached regular request building stage",
))
}
sgl-model-gateway/src/routers/grpc/regular/stages/response_processing.rs
+1
View File
@@ -49,6 +49,7 @@ impl PipelineStage for ResponseProcessingStage {
"RequestType::Responses reached regular response processing stage"
);
Err(error::internal_error(
"responses_in_wrong_pipeline",
"RequestType::Responses reached regular response processing stage",
))
}
sgl-model-gateway/src/routers/grpc/utils.rs
+16 -11
View File
@@ -52,14 +52,20 @@ pub async fn get_grpc_client_from_worker(worker: &Arc<dyn Worker>) -> Result<Grp
error = %e,
"Failed to get gRPC client from worker"
);
error::internal_error(format!("Failed to get gRPC client: {}", e))
error::internal_error(
"get_grpc_client_failed",
format!("Failed to get gRPC client: {}", e),
)
})?
.ok_or_else(|| {
error!(
function = "get_grpc_client_from_worker",
"Selected worker not configured for gRPC"
);
error::internal_error("Selected worker is not configured for gRPC")
error::internal_error(
"worker_not_configured_for_grpc",
"Selected worker is not configured for gRPC",
)
})?;
Ok((*client_arc).clone())
@@ -612,11 +618,10 @@ pub async fn collect_stream_responses(
ProtoResponseVariant::Error(err) => {
error!(function = "collect_stream_responses", worker = %worker_name, error = %err.message(), "Worker generation error");
// Don't mark as completed - let Drop send abort for error cases
return Err(error::internal_error(format!(
"{} generation failed: {}",
worker_name,
err.message()
)));
return Err(error::internal_error(
"worker_generation_failed",
format!("{} generation failed: {}", worker_name, err.message()),
));
}
ProtoResponseVariant::Chunk(_chunk) => {
// Streaming chunk - no action needed
@@ -629,10 +634,10 @@ pub async fn collect_stream_responses(
Err(e) => {
error!(function = "collect_stream_responses", worker = %worker_name, error = ?e, "Worker stream error");
// Don't mark as completed - let Drop send abort for error cases
return Err(error::internal_error(format!(
"{} stream failed: {}",
worker_name, e
)));
return Err(error::internal_error(
"worker_stream_failed",
format!("{} stream failed: {}", worker_name, e),
));
}
}
}