diff --git a/sgl-model-gateway/src/observability/metrics.rs b/sgl-model-gateway/src/observability/metrics.rs index d57d5a52e..64a39a73b 100644 --- a/sgl-model-gateway/src/observability/metrics.rs +++ b/sgl-model-gateway/src/observability/metrics.rs @@ -318,8 +318,7 @@ impl RouterMetrics { } pub fn record_request_duration(duration: Duration) { - histogram!("sgl_router_request_duration_seconds") - .record(duration.as_secs_f64()); + histogram!("sgl_router_request_duration_seconds").record(duration.as_secs_f64()); } pub fn record_request_error(route: &str, error_type: &str) { diff --git a/sgl-model-gateway/src/routers/grpc/error.rs b/sgl-model-gateway/src/routers/error.rs similarity index 100% rename from sgl-model-gateway/src/routers/grpc/error.rs rename to sgl-model-gateway/src/routers/error.rs diff --git a/sgl-model-gateway/src/routers/grpc/common/response_collection.rs b/sgl-model-gateway/src/routers/grpc/common/response_collection.rs index 68daa138f..2c4af123f 100644 --- a/sgl-model-gateway/src/routers/grpc/common/response_collection.rs +++ b/sgl-model-gateway/src/routers/grpc/common/response_collection.rs @@ -5,8 +5,9 @@ use axum::response::Response; -use crate::routers::grpc::{ - context::ExecutionResult, error, proto_wrapper::ProtoGenerateComplete, utils, +use crate::routers::{ + error, + grpc::{context::ExecutionResult, proto_wrapper::ProtoGenerateComplete, utils}, }; /// Collect and merge responses from execution result diff --git a/sgl-model-gateway/src/routers/grpc/common/responses/utils.rs b/sgl-model-gateway/src/routers/grpc/common/responses/utils.rs index 3fbe20efb..e85aab456 100644 --- a/sgl-model-gateway/src/routers/grpc/common/responses/utils.rs +++ b/sgl-model-gateway/src/routers/grpc/common/responses/utils.rs @@ -18,7 +18,7 @@ use crate::{ responses::{ResponseTool, ResponseToolType, ResponsesRequest, ResponsesResponse}, }, routers::{ - grpc::error, + error, openai::{conversations::persist_conversation_items, mcp::ensure_request_mcp_client}, }, }; diff --git a/sgl-model-gateway/src/routers/grpc/common/stages/client_acquisition.rs b/sgl-model-gateway/src/routers/grpc/common/stages/client_acquisition.rs index 38ac075aa..ca4661992 100644 --- a/sgl-model-gateway/src/routers/grpc/common/stages/client_acquisition.rs +++ b/sgl-model-gateway/src/routers/grpc/common/stages/client_acquisition.rs @@ -5,9 +5,12 @@ use axum::response::Response; use tracing::error; use super::PipelineStage; -use crate::routers::grpc::{ - context::{ClientSelection, RequestContext, WorkerSelection}, - error, utils, +use crate::routers::{ + error, + grpc::{ + context::{ClientSelection, RequestContext, WorkerSelection}, + utils, + }, }; /// Client acquisition stage: Get gRPC clients from selected workers diff --git a/sgl-model-gateway/src/routers/grpc/common/stages/dispatch_metadata.rs b/sgl-model-gateway/src/routers/grpc/common/stages/dispatch_metadata.rs index 3f63782e9..c14b0c972 100644 --- a/sgl-model-gateway/src/routers/grpc/common/stages/dispatch_metadata.rs +++ b/sgl-model-gateway/src/routers/grpc/common/stages/dispatch_metadata.rs @@ -7,9 +7,9 @@ use axum::response::Response; use tracing::error; use super::PipelineStage; -use crate::routers::grpc::{ - context::{DispatchMetadata, RequestContext, RequestType, WorkerSelection}, +use crate::routers::{ error, + grpc::context::{DispatchMetadata, RequestContext, RequestType, WorkerSelection}, }; /// Dispatch metadata stage: Prepare metadata for dispatch diff --git a/sgl-model-gateway/src/routers/grpc/common/stages/request_execution.rs b/sgl-model-gateway/src/routers/grpc/common/stages/request_execution.rs index 89898451d..558534e7d 100644 --- a/sgl-model-gateway/src/routers/grpc/common/stages/request_execution.rs +++ b/sgl-model-gateway/src/routers/grpc/common/stages/request_execution.rs @@ -5,10 +5,12 @@ use axum::response::Response; use tracing::{error, info_span, Instrument}; use super::PipelineStage; -use crate::routers::grpc::{ - context::{ClientSelection, ExecutionResult, RequestContext}, +use crate::routers::{ error, - proto_wrapper::{ProtoGenerateRequest, ProtoStream}, + grpc::{ + context::{ClientSelection, ExecutionResult, RequestContext}, + proto_wrapper::{ProtoGenerateRequest, ProtoStream}, + }, }; type StreamResult = Result>; diff --git a/sgl-model-gateway/src/routers/grpc/common/stages/worker_selection.rs b/sgl-model-gateway/src/routers/grpc/common/stages/worker_selection.rs index f043f57f8..9ae600a22 100644 --- a/sgl-model-gateway/src/routers/grpc/common/stages/worker_selection.rs +++ b/sgl-model-gateway/src/routers/grpc/common/stages/worker_selection.rs @@ -10,9 +10,9 @@ use super::PipelineStage; use crate::{ core::{ConnectionMode, Worker, WorkerRegistry, WorkerType}, policies::PolicyRegistry, - routers::grpc::{ - context::{RequestContext, WorkerSelection}, + routers::{ error, + grpc::context::{RequestContext, WorkerSelection}, }, }; diff --git a/sgl-model-gateway/src/routers/grpc/harmony/processor.rs b/sgl-model-gateway/src/routers/grpc/harmony/processor.rs index ad7556574..b0c84b9e0 100644 --- a/sgl-model-gateway/src/routers/grpc/harmony/processor.rs +++ b/sgl-model-gateway/src/routers/grpc/harmony/processor.rs @@ -16,10 +16,12 @@ use crate::{ ResponseStatus, ResponseUsage, ResponsesRequest, ResponsesResponse, ResponsesUsage, }, }, - routers::grpc::{ - common::{response_collection, response_formatting}, - context::{DispatchMetadata, ExecutionResult}, + routers::{ error, + grpc::{ + common::{response_collection, response_formatting}, + context::{DispatchMetadata, ExecutionResult}, + }, }, }; diff --git a/sgl-model-gateway/src/routers/grpc/harmony/responses.rs b/sgl-model-gateway/src/routers/grpc/harmony/responses.rs index 559ad5a24..94f8feced 100644 --- a/sgl-model-gateway/src/routers/grpc/harmony/responses.rs +++ b/sgl-model-gateway/src/routers/grpc/harmony/responses.rs @@ -54,15 +54,17 @@ use crate::{ ResponsesUsage, StringOrContentParts, }, }, - routers::grpc::{ - common::responses::{ - build_sse_response, ensure_mcp_connection, persist_response_if_needed, - streaming::{OutputItemType, ResponseStreamEventEmitter}, - }, - context::SharedComponents, + routers::{ error, - harmony::{processor::ResponsesIterationResult, streaming::HarmonyStreamingProcessor}, - pipeline::RequestPipeline, + grpc::{ + common::responses::{ + build_sse_response, ensure_mcp_connection, persist_response_if_needed, + streaming::{OutputItemType, ResponseStreamEventEmitter}, + }, + context::SharedComponents, + harmony::{processor::ResponsesIterationResult, streaming::HarmonyStreamingProcessor}, + pipeline::RequestPipeline, + }, }, }; diff --git a/sgl-model-gateway/src/routers/grpc/harmony/stages/preparation.rs b/sgl-model-gateway/src/routers/grpc/harmony/stages/preparation.rs index c370514cb..0a05f65ac 100644 --- a/sgl-model-gateway/src/routers/grpc/harmony/stages/preparation.rs +++ b/sgl-model-gateway/src/routers/grpc/harmony/stages/preparation.rs @@ -12,10 +12,13 @@ use crate::{ common::{Tool, ToolChoice, ToolChoiceValue}, responses::ResponsesRequest, }, - routers::grpc::{ - common::{responses::utils::extract_tools_from_response_tools, stages::PipelineStage}, - context::{PreparationOutput, RequestContext, RequestType}, - error, utils, + routers::{ + error, + grpc::{ + common::{responses::utils::extract_tools_from_response_tools, stages::PipelineStage}, + context::{PreparationOutput, RequestContext, RequestType}, + utils, + }, }, }; diff --git a/sgl-model-gateway/src/routers/grpc/harmony/stages/request_building.rs b/sgl-model-gateway/src/routers/grpc/harmony/stages/request_building.rs index 7d8fa0957..199c36813 100644 --- a/sgl-model-gateway/src/routers/grpc/harmony/stages/request_building.rs +++ b/sgl-model-gateway/src/routers/grpc/harmony/stages/request_building.rs @@ -5,11 +5,13 @@ use axum::response::Response; use tracing::{debug, error}; use uuid::Uuid; -use crate::routers::grpc::{ - common::stages::{helpers, PipelineStage}, - context::{ClientSelection, RequestContext, RequestType, WorkerSelection}, +use crate::routers::{ error, - proto_wrapper::ProtoGenerateRequest, + grpc::{ + common::stages::{helpers, PipelineStage}, + context::{ClientSelection, RequestContext, RequestType, WorkerSelection}, + proto_wrapper::ProtoGenerateRequest, + }, }; /// Harmony Request Building stage: Convert Harmony tokens to gRPC request diff --git a/sgl-model-gateway/src/routers/grpc/harmony/stages/response_processing.rs b/sgl-model-gateway/src/routers/grpc/harmony/stages/response_processing.rs index 924556e34..35161c22d 100644 --- a/sgl-model-gateway/src/routers/grpc/harmony/stages/response_processing.rs +++ b/sgl-model-gateway/src/routers/grpc/harmony/stages/response_processing.rs @@ -7,10 +7,12 @@ use axum::response::Response; use tracing::error; use super::super::{HarmonyResponseProcessor, HarmonyStreamingProcessor}; -use crate::routers::grpc::{ - common::stages::PipelineStage, - context::{FinalResponse, RequestContext, RequestType}, +use crate::routers::{ error, + grpc::{ + common::stages::PipelineStage, + context::{FinalResponse, RequestContext, RequestType}, + }, }; /// Harmony Response Processing stage: Parse and format Harmony responses diff --git a/sgl-model-gateway/src/routers/grpc/mod.rs b/sgl-model-gateway/src/routers/grpc/mod.rs index 58b017bff..30b2bc7d8 100644 --- a/sgl-model-gateway/src/routers/grpc/mod.rs +++ b/sgl-model-gateway/src/routers/grpc/mod.rs @@ -5,7 +5,6 @@ use crate::{grpc_client::sglang_proto::MultimodalInputs, protocols::common::Stri pub mod client; pub mod common; pub mod context; -pub mod error; pub mod harmony; pub mod pd_router; pub mod pipeline; diff --git a/sgl-model-gateway/src/routers/grpc/pipeline.rs b/sgl-model-gateway/src/routers/grpc/pipeline.rs index cb09f2a50..5ac69638b 100644 --- a/sgl-model-gateway/src/routers/grpc/pipeline.rs +++ b/sgl-model-gateway/src/routers/grpc/pipeline.rs @@ -11,7 +11,7 @@ use tracing::error; use super::{ common::stages::*, context::*, - error, harmony, + harmony, regular::{processor, stages::*, streaming}, }; use crate::{ @@ -22,6 +22,7 @@ use crate::{ generate::GenerateRequest, }, reasoning_parser::ParserFactory as ReasoningParserFactory, + routers::error, tokenizer::traits::Tokenizer, tool_parser::ParserFactory as ToolParserFactory, }; diff --git a/sgl-model-gateway/src/routers/grpc/regular/processor.rs b/sgl-model-gateway/src/routers/grpc/regular/processor.rs index 49dabd1e6..8f14cd819 100644 --- a/sgl-model-gateway/src/routers/grpc/regular/processor.rs +++ b/sgl-model-gateway/src/routers/grpc/regular/processor.rs @@ -16,12 +16,14 @@ use crate::{ generate::{GenerateMetaInfo, GenerateRequest, GenerateResponse}, }, reasoning_parser::ParserFactory as ReasoningParserFactory, - routers::grpc::{ - common::{response_collection, response_formatting}, - context::{DispatchMetadata, ExecutionResult}, + routers::{ error, - proto_wrapper::ProtoGenerateComplete, - utils, + grpc::{ + common::{response_collection, response_formatting}, + context::{DispatchMetadata, ExecutionResult}, + proto_wrapper::ProtoGenerateComplete, + utils, + }, }, tokenizer::{ stop::{SequenceDecoderOutput, StopSequenceDecoder}, diff --git a/sgl-model-gateway/src/routers/grpc/regular/responses/handlers.rs b/sgl-model-gateway/src/routers/grpc/regular/responses/handlers.rs index 967d06c62..eeed1cc40 100644 --- a/sgl-model-gateway/src/routers/grpc/regular/responses/handlers.rs +++ b/sgl-model-gateway/src/routers/grpc/regular/responses/handlers.rs @@ -64,12 +64,12 @@ use crate::{ ResponsesUsage, }, }, - routers::grpc::{ - common::responses::{ + routers::{ + error, + grpc::common::responses::{ build_sse_response, ensure_mcp_connection, persist_response_if_needed, streaming::ResponseStreamEventEmitter, }, - error, }, }; diff --git a/sgl-model-gateway/src/routers/grpc/regular/responses/tool_loop.rs b/sgl-model-gateway/src/routers/grpc/regular/responses/tool_loop.rs index 29e274e87..4cfab3fce 100644 --- a/sgl-model-gateway/src/routers/grpc/regular/responses/tool_loop.rs +++ b/sgl-model-gateway/src/routers/grpc/regular/responses/tool_loop.rs @@ -34,9 +34,9 @@ use crate::{ ResponsesResponse, }, }, - routers::grpc::{ - common::responses::streaming::{OutputItemType, ResponseStreamEventEmitter}, + routers::{ error, + grpc::common::responses::streaming::{OutputItemType, ResponseStreamEventEmitter}, }, }; diff --git a/sgl-model-gateway/src/routers/grpc/regular/stages/chat/preparation.rs b/sgl-model-gateway/src/routers/grpc/regular/stages/chat/preparation.rs index 09be7f1c6..adb48ab52 100644 --- a/sgl-model-gateway/src/routers/grpc/regular/stages/chat/preparation.rs +++ b/sgl-model-gateway/src/routers/grpc/regular/stages/chat/preparation.rs @@ -8,10 +8,13 @@ use tracing::error; use crate::{ protocols::chat::ChatCompletionRequest, - routers::grpc::{ - common::stages::PipelineStage, - context::{PreparationOutput, RequestContext}, - error, utils, + routers::{ + error, + grpc::{ + common::stages::PipelineStage, + context::{PreparationOutput, RequestContext}, + utils, + }, }, }; diff --git a/sgl-model-gateway/src/routers/grpc/regular/stages/chat/request_building.rs b/sgl-model-gateway/src/routers/grpc/regular/stages/chat/request_building.rs index 40cbd96d5..2a88b3cad 100644 --- a/sgl-model-gateway/src/routers/grpc/regular/stages/chat/request_building.rs +++ b/sgl-model-gateway/src/routers/grpc/regular/stages/chat/request_building.rs @@ -5,12 +5,14 @@ use axum::response::Response; use tracing::error; use uuid::Uuid; -use crate::routers::grpc::{ - client::GrpcClient, - common::stages::{helpers, PipelineStage}, - context::{ClientSelection, RequestContext, WorkerSelection}, +use crate::routers::{ error, - proto_wrapper::ProtoGenerateRequest, + grpc::{ + client::GrpcClient, + common::stages::{helpers, PipelineStage}, + context::{ClientSelection, RequestContext, WorkerSelection}, + proto_wrapper::ProtoGenerateRequest, + }, }; /// Chat request building stage diff --git a/sgl-model-gateway/src/routers/grpc/regular/stages/chat/response_processing.rs b/sgl-model-gateway/src/routers/grpc/regular/stages/chat/response_processing.rs index 410ee817b..42af1a45e 100644 --- a/sgl-model-gateway/src/routers/grpc/regular/stages/chat/response_processing.rs +++ b/sgl-model-gateway/src/routers/grpc/regular/stages/chat/response_processing.rs @@ -9,11 +9,13 @@ use async_trait::async_trait; use axum::response::Response; use tracing::error; -use crate::routers::grpc::{ - common::stages::PipelineStage, - context::{FinalResponse, RequestContext}, +use crate::routers::{ error, - regular::{processor, streaming}, + grpc::{ + common::stages::PipelineStage, + context::{FinalResponse, RequestContext}, + regular::{processor, streaming}, + }, }; /// Chat response processing stage diff --git a/sgl-model-gateway/src/routers/grpc/regular/stages/generate/preparation.rs b/sgl-model-gateway/src/routers/grpc/regular/stages/generate/preparation.rs index ebd4fdf12..dadcffcfc 100644 --- a/sgl-model-gateway/src/routers/grpc/regular/stages/generate/preparation.rs +++ b/sgl-model-gateway/src/routers/grpc/regular/stages/generate/preparation.rs @@ -8,10 +8,13 @@ use tracing::error; use crate::{ protocols::{common::InputIds, generate::GenerateRequest}, - routers::grpc::{ - common::stages::PipelineStage, - context::{PreparationOutput, RequestContext}, - error, utils, + routers::{ + error, + grpc::{ + common::stages::PipelineStage, + context::{PreparationOutput, RequestContext}, + utils, + }, }, tokenizer::traits::Tokenizer, }; diff --git a/sgl-model-gateway/src/routers/grpc/regular/stages/generate/request_building.rs b/sgl-model-gateway/src/routers/grpc/regular/stages/generate/request_building.rs index 3ffb212ff..216312f5b 100644 --- a/sgl-model-gateway/src/routers/grpc/regular/stages/generate/request_building.rs +++ b/sgl-model-gateway/src/routers/grpc/regular/stages/generate/request_building.rs @@ -5,12 +5,14 @@ use axum::response::Response; use tracing::error; use uuid::Uuid; -use crate::routers::grpc::{ - client::GrpcClient, - common::stages::{helpers, PipelineStage}, - context::{ClientSelection, RequestContext, WorkerSelection}, +use crate::routers::{ error, - proto_wrapper::ProtoGenerateRequest, + grpc::{ + client::GrpcClient, + common::stages::{helpers, PipelineStage}, + context::{ClientSelection, RequestContext, WorkerSelection}, + proto_wrapper::ProtoGenerateRequest, + }, }; /// Generate request building stage diff --git a/sgl-model-gateway/src/routers/grpc/regular/stages/generate/response_processing.rs b/sgl-model-gateway/src/routers/grpc/regular/stages/generate/response_processing.rs index 3f325ac0d..7afd626d4 100644 --- a/sgl-model-gateway/src/routers/grpc/regular/stages/generate/response_processing.rs +++ b/sgl-model-gateway/src/routers/grpc/regular/stages/generate/response_processing.rs @@ -6,11 +6,13 @@ use async_trait::async_trait; use axum::response::Response; use tracing::error; -use crate::routers::grpc::{ - common::stages::PipelineStage, - context::{FinalResponse, RequestContext}, +use crate::routers::{ error, - regular::{processor, streaming}, + grpc::{ + common::stages::PipelineStage, + context::{FinalResponse, RequestContext}, + regular::{processor, streaming}, + }, }; /// Generate response processing stage diff --git a/sgl-model-gateway/src/routers/grpc/regular/stages/preparation.rs b/sgl-model-gateway/src/routers/grpc/regular/stages/preparation.rs index b0050867b..b2d8c1c03 100644 --- a/sgl-model-gateway/src/routers/grpc/regular/stages/preparation.rs +++ b/sgl-model-gateway/src/routers/grpc/regular/stages/preparation.rs @@ -8,10 +8,12 @@ use axum::response::Response; use tracing::error; use super::{chat::ChatPreparationStage, generate::GeneratePreparationStage}; -use crate::routers::grpc::{ - common::stages::PipelineStage, - context::{RequestContext, RequestType}, +use crate::routers::{ error as grpc_error, + grpc::{ + common::stages::PipelineStage, + context::{RequestContext, RequestType}, + }, }; /// Preparation stage (delegates to endpoint-specific implementations) diff --git a/sgl-model-gateway/src/routers/grpc/regular/stages/request_building.rs b/sgl-model-gateway/src/routers/grpc/regular/stages/request_building.rs index 4c7630735..4f1ee62c9 100644 --- a/sgl-model-gateway/src/routers/grpc/regular/stages/request_building.rs +++ b/sgl-model-gateway/src/routers/grpc/regular/stages/request_building.rs @@ -5,10 +5,12 @@ use axum::response::Response; use tracing::error; use super::{chat::ChatRequestBuildingStage, generate::GenerateRequestBuildingStage}; -use crate::routers::grpc::{ - common::stages::PipelineStage, - context::{RequestContext, RequestType}, +use crate::routers::{ error as grpc_error, + grpc::{ + common::stages::PipelineStage, + context::{RequestContext, RequestType}, + }, }; /// Request building stage (delegates to endpoint-specific implementations) diff --git a/sgl-model-gateway/src/routers/grpc/regular/stages/response_processing.rs b/sgl-model-gateway/src/routers/grpc/regular/stages/response_processing.rs index a5cb5a135..2dea97e08 100644 --- a/sgl-model-gateway/src/routers/grpc/regular/stages/response_processing.rs +++ b/sgl-model-gateway/src/routers/grpc/regular/stages/response_processing.rs @@ -7,11 +7,13 @@ use axum::response::Response; use tracing::error; use super::{chat::ChatResponseProcessingStage, generate::GenerateResponseProcessingStage}; -use crate::routers::grpc::{ - common::stages::PipelineStage, - context::{RequestContext, RequestType}, +use crate::routers::{ error, - regular::{processor, streaming}, + grpc::{ + common::stages::PipelineStage, + context::{RequestContext, RequestType}, + regular::{processor, streaming}, + }, }; /// Response processing stage (delegates to endpoint-specific implementations) diff --git a/sgl-model-gateway/src/routers/grpc/utils.rs b/sgl-model-gateway/src/routers/grpc/utils.rs index e470aec01..1f199cbfd 100644 --- a/sgl-model-gateway/src/routers/grpc/utils.rs +++ b/sgl-model-gateway/src/routers/grpc/utils.rs @@ -9,7 +9,6 @@ use uuid::Uuid; use super::{ client::GrpcClient, - error, proto_wrapper::{ProtoGenerateComplete, ProtoStream}, ProcessedMessages, }; @@ -28,7 +27,7 @@ use crate::{ ParserFactory as ReasoningParserFactory, PooledParser as ReasoningPooledParser, ReasoningParser, }, - routers::grpc::proto_wrapper::ProtoResponseVariant, + routers::{error, grpc::proto_wrapper::ProtoResponseVariant}, tokenizer::{ cache::CachedTokenizer, chat_template::{ChatTemplateContentFormat, ChatTemplateParams}, diff --git a/sgl-model-gateway/src/routers/mod.rs b/sgl-model-gateway/src/routers/mod.rs index fa2ba4084..6ee0f3bf4 100644 --- a/sgl-model-gateway/src/routers/mod.rs +++ b/sgl-model-gateway/src/routers/mod.rs @@ -21,6 +21,7 @@ use crate::protocols::{ }; pub mod conversations; +pub mod error; pub mod factory; pub mod grpc; pub mod header_utils;