chenchenghao/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[router] cleaned up all the redundant comments in the config module (#12147)

Browse Source
This commit is contained in:
Chang Su
2025-10-25 23:10:59 -07:00
committed by GitHub
parent 03e7d94916
commit 773d89da8c
4 changed files with 62 additions and 279 deletions
Download Patch File Download Diff File Expand all files Collapse all files

150
sgl-router/src/config/builder.rs
View File

@@ -18,12 +18,11 @@ pub struct RouterConfigBuilder {
}
impl RouterConfigBuilder {
/// Create a new builder with default values
pub fn new() -> Self {
Self::default()
}
/// Create a builder from an existing configuration (takes ownership)
/// Takes ownership
pub fn from_config(config: RouterConfig) -> Self {
Self {
config,
@@ -34,20 +33,17 @@ impl RouterConfigBuilder {
}
}
/// Create a builder from a reference to an existing configuration
pub fn from_config_ref(config: &RouterConfig) -> Self {
Self::from_config(config.clone())
}
// ==================== Routing Mode Setters ====================
// ==================== Routing Mode ====================
/// Set regular routing mode with worker URLs
pub fn regular_mode(mut self, worker_urls: Vec<String>) -> Self {
self.config.mode = RoutingMode::Regular { worker_urls };
self
}
/// Set prefill-decode routing mode
pub fn prefill_decode_mode(
mut self,
prefill_urls: Vec<(String, Option<u16>)>,
@@ -62,7 +58,7 @@ impl RouterConfigBuilder {
self
}
/// Set prefill-decode mode with separate policies
/// With separate policies
pub fn prefill_decode_mode_with_policies(
mut self,
prefill_urls: Vec<(String, Option<u16>)>,
@@ -79,39 +75,33 @@ impl RouterConfigBuilder {
self
}
/// Set OpenAI routing mode
pub fn openai_mode(mut self, worker_urls: Vec<String>) -> Self {
self.config.mode = RoutingMode::OpenAI { worker_urls };
self
}
/// Set the routing mode directly
pub fn mode(mut self, mode: RoutingMode) -> Self {
self.config.mode = mode;
self
}
// ==================== Policy Setters ====================
// ==================== Policy ====================
/// Set the routing policy
pub fn policy(mut self, policy: PolicyConfig) -> Self {
self.config.policy = policy;
self
}
/// Set random policy
pub fn random_policy(mut self) -> Self {
self.config.policy = PolicyConfig::Random;
self
}
/// Set round-robin policy
pub fn round_robin_policy(mut self) -> Self {
self.config.policy = PolicyConfig::RoundRobin;
self
}
/// Set cache-aware policy with parameters
pub fn cache_aware_policy(
mut self,
cache_threshold: f32,
@@ -130,7 +120,6 @@ impl RouterConfigBuilder {
self
}
/// Set power-of-two policy
pub fn power_of_two_policy(mut self, load_check_interval_secs: u64) -> Self {
self.config.policy = PolicyConfig::PowerOfTwo {
load_check_interval_secs,
@@ -138,65 +127,55 @@ impl RouterConfigBuilder {
self
}
// ==================== Connection Settings ====================
// ==================== Connection ====================
/// Set connection mode
pub fn connection_mode(mut self, mode: ConnectionMode) -> Self {
self.config.connection_mode = mode;
self
}
/// Set HTTP connection mode
pub fn http_connection(mut self) -> Self {
self.config.connection_mode = ConnectionMode::Http;
self
}
/// Set gRPC connection mode with optional port
pub fn grpc_connection(mut self, port: Option<u16>) -> Self {
self.config.connection_mode = ConnectionMode::Grpc { port };
self
}
/// Set gRPC connection mode without specifying a port
pub fn grpc_connection_default(mut self) -> Self {
self.config.connection_mode = ConnectionMode::Grpc { port: None };
self
}
/// Set host address
pub fn host<S: Into<String>>(mut self, host: S) -> Self {
self.config.host = host.into();
self
}
/// Set port number
pub fn port(mut self, port: u16) -> Self {
self.config.port = port;
self
}
// ==================== Request Settings ====================
// ==================== Request ====================
/// Set maximum payload size in bytes
pub fn max_payload_size(mut self, size: usize) -> Self {
self.config.max_payload_size = size;
self
}
/// Set request timeout in seconds
pub fn request_timeout_secs(mut self, timeout: u64) -> Self {
self.config.request_timeout_secs = timeout;
self
}
/// Set worker startup timeout in seconds
pub fn worker_startup_timeout_secs(mut self, timeout: u64) -> Self {
self.config.worker_startup_timeout_secs = timeout;
self
}
/// Set worker startup check interval in seconds
pub fn worker_startup_check_interval_secs(mut self, interval: u64) -> Self {
self.config.worker_startup_check_interval_secs = interval;
self
@@ -204,31 +183,26 @@ impl RouterConfigBuilder {
// ==================== Rate Limiting ====================
/// Set maximum concurrent requests
pub fn max_concurrent_requests(mut self, max: i32) -> Self {
self.config.max_concurrent_requests = max;
self
}
/// Disable rate limiting
pub fn disable_rate_limiting(mut self) -> Self {
self.config.max_concurrent_requests = -1;
self
}
/// Set queue size for pending requests
pub fn queue_size(mut self, size: usize) -> Self {
self.config.queue_size = size;
self
}
/// Set queue timeout in seconds
pub fn queue_timeout_secs(mut self, timeout: u64) -> Self {
self.config.queue_timeout_secs = timeout;
self
}
/// Set rate limit tokens per second
pub fn rate_limit_tokens_per_second(mut self, tokens: i32) -> Self {
self.config.rate_limit_tokens_per_second = Some(tokens);
self
@@ -236,81 +210,70 @@ impl RouterConfigBuilder {
// ==================== Security & CORS ====================
/// Set API key for worker authorization
pub fn api_key<S: Into<String>>(mut self, key: S) -> Self {
self.config.api_key = Some(key.into());
self
}
/// Set CORS allowed origins
pub fn cors_allowed_origins(mut self, origins: Vec<String>) -> Self {
self.config.cors_allowed_origins = origins;
self
}
/// Add a single CORS origin
pub fn add_cors_origin<S: Into<String>>(mut self, origin: S) -> Self {
self.config.cors_allowed_origins.push(origin.into());
self
}
// ==================== Retry Configuration ====================
// ==================== Retry ====================
/// Set retry configuration
pub fn retry_config(mut self, retry: RetryConfig) -> Self {
self.config.retry = retry;
self
}
/// Disable retries
pub fn disable_retries(mut self) -> Self {
self.config.disable_retries = true;
self
}
/// Enable retries
pub fn enable_retries(mut self) -> Self {
self.config.disable_retries = false;
self
}
// ==================== Circuit Breaker Configuration ====================
// ==================== Circuit Breaker ====================
/// Set circuit breaker configuration
pub fn circuit_breaker_config(mut self, circuit_breaker: CircuitBreakerConfig) -> Self {
self.config.circuit_breaker = circuit_breaker;
self
}
/// Disable circuit breaker
pub fn disable_circuit_breaker(mut self) -> Self {
self.config.disable_circuit_breaker = true;
self
}
/// Enable circuit breaker
pub fn enable_circuit_breaker(mut self) -> Self {
self.config.disable_circuit_breaker = false;
self
}
// ==================== Health Check Configuration ====================
// ==================== Health Check ====================
/// Set health check configuration
pub fn health_check_config(mut self, health_check: HealthCheckConfig) -> Self {
self.config.health_check = health_check;
self
}
// ==================== Discovery Configuration ====================
// ==================== Discovery ====================
/// Set service discovery configuration
pub fn discovery_config(mut self, discovery: DiscoveryConfig) -> Self {
self.config.discovery = Some(discovery);
self
}
/// Enable service discovery with default settings
/// With default settings
pub fn enable_discovery(mut self) -> Self {
self.config.discovery = Some(DiscoveryConfig {
enabled: true,
@@ -319,15 +282,13 @@ impl RouterConfigBuilder {
self
}
// ==================== Metrics Configuration ====================
// ==================== Metrics ====================
/// Set metrics configuration
pub fn metrics_config(mut self, metrics: MetricsConfig) -> Self {
self.config.metrics = Some(metrics);
self
}
/// Enable metrics with host and port
pub fn enable_metrics<S: Into<String>>(mut self, host: S, port: u16) -> Self {
self.config.metrics = Some(MetricsConfig {
host: host.into(),
@@ -336,115 +297,100 @@ impl RouterConfigBuilder {
self
}
// ==================== Logging Configuration ====================
// ==================== Logging ====================
/// Set log directory
pub fn log_dir<S: Into<String>>(mut self, dir: S) -> Self {
self.config.log_dir = Some(dir.into());
self
}
/// Set log level
pub fn log_level<S: Into<String>>(mut self, level: S) -> Self {
self.config.log_level = Some(level.into());
self
}
/// Set custom request ID headers
pub fn request_id_headers(mut self, headers: Vec<String>) -> Self {
self.config.request_id_headers = Some(headers);
self
}
// ==================== IGW Mode Configuration ====================
// ==================== IGW Mode ====================
/// Enable Inference Gateway mode
pub fn enable_igw(mut self) -> Self {
self.config.enable_igw = true;
self
}
/// Disable Inference Gateway mode (use proxy mode)
/// Use proxy mode
pub fn disable_igw(mut self) -> Self {
self.config.enable_igw = false;
self
}
/// Set model path for tokenizer
pub fn model_path<S: Into<String>>(mut self, path: S) -> Self {
self.config.model_path = Some(path.into());
self
}
/// Set tokenizer path (overrides model_path tokenizer)
/// Overrides model_path tokenizer
pub fn tokenizer_path<S: Into<String>>(mut self, path: S) -> Self {
self.config.tokenizer_path = Some(path.into());
self
}
/// Set chat template path
pub fn chat_template<S: Into<String>>(mut self, path: S) -> Self {
self.config.chat_template = Some(path.into());
self
}
// ==================== History Backend Configuration ====================
// ==================== History Backend ====================
/// Set history backend
pub fn history_backend(mut self, backend: HistoryBackend) -> Self {
self.config.history_backend = backend;
self
}
/// Use memory history backend
pub fn memory_history(mut self) -> Self {
self.config.history_backend = HistoryBackend::Memory;
self
}
/// Disable history storage
pub fn no_history(mut self) -> Self {
self.config.history_backend = HistoryBackend::None;
self
}
/// Use Oracle history backend
pub fn oracle_history(mut self, oracle_config: OracleConfig) -> Self {
self.config.history_backend = HistoryBackend::Oracle;
self.config.oracle = Some(oracle_config);
self
}
// ==================== Parsers Configuration ====================
// ==================== Parsers ====================
/// Set reasoning parser
pub fn reasoning_parser<S: Into<String>>(mut self, parser: S) -> Self {
self.config.reasoning_parser = Some(parser.into());
self
}
/// Set tool call parser
pub fn tool_call_parser<S: Into<String>>(mut self, parser: S) -> Self {
self.config.tool_call_parser = Some(parser.into());
self
}
// ==================== Tokenizer Cache Configuration ====================
// ==================== Tokenizer Cache ====================
/// Set tokenizer cache configuration
pub fn tokenizer_cache(mut self, cache: TokenizerCacheConfig) -> Self {
self.config.tokenizer_cache = cache;
self
}
/// Enable L0 cache with entry limit
pub fn enable_l0_cache(mut self, max_entries: usize) -> Self {
self.config.tokenizer_cache.enable_l0 = true;
self.config.tokenizer_cache.l0_max_entries = max_entries;
self
}
/// Enable L1 cache with memory limit
pub fn enable_l1_cache(mut self, max_memory: usize) -> Self {
self.config.tokenizer_cache.enable_l1 = true;
self.config.tokenizer_cache.l1_max_memory = max_memory;
@@ -453,51 +399,44 @@ impl RouterConfigBuilder {
// ==================== Data Parallelism ====================
/// Enable data parallelism aware scheduling
pub fn enable_dp_aware(mut self) -> Self {
self.config.dp_aware = true;
self
}
/// Disable data parallelism aware scheduling
pub fn disable_dp_aware(mut self) -> Self {
self.config.dp_aware = false;
self
}
// ==================== Conditional Boolean Setters ====================
// These methods accept bool parameters to conditionally set flags,
// eliminating the need for if statements in calling code
// ==================== Boolean Setters ====================
// Accept bool parameters to conditionally set flags without if statements
/// Set dp_aware flag conditionally
pub fn dp_aware(mut self, enable: bool) -> Self {
self.config.dp_aware = enable;
self
}
/// Enable or disable retries (inverse of disable_retries field)
/// Inverse of disable_retries field
pub fn retries(mut self, enable: bool) -> Self {
self.config.disable_retries = !enable;
self
}
/// Enable or disable circuit breaker (inverse of disable_circuit_breaker field)
/// Inverse of disable_circuit_breaker field
pub fn circuit_breaker(mut self, enable: bool) -> Self {
self.config.disable_circuit_breaker = !enable;
self
}
/// Set enable_igw flag conditionally
pub fn igw(mut self, enable: bool) -> Self {
self.config.enable_igw = enable;
self
}
// ==================== Option-Aware Setters ====================
// These methods accept Option<T> and only set if Some, making it easier
// to conditionally set values without if-let chains
// ==================== Option Setters ====================
// Accept Option<T> and only set if Some
/// Set API key if Some
pub fn maybe_api_key(mut self, key: Option<impl Into<String>>) -> Self {
if let Some(k) = key {
self.config.api_key = Some(k.into());
@@ -505,61 +444,51 @@ impl RouterConfigBuilder {
self
}
/// Set discovery config if Some
pub fn maybe_discovery(mut self, discovery: Option<DiscoveryConfig>) -> Self {
self.config.discovery = discovery;
self
}
/// Set metrics config if Some
pub fn maybe_metrics(mut self, metrics: Option<MetricsConfig>) -> Self {
self.config.metrics = metrics;
self
}
/// Set log directory if Some
pub fn maybe_log_dir(mut self, dir: Option<impl Into<String>>) -> Self {
self.config.log_dir = dir.map(|d| d.into());
self
}
/// Set log level if Some
pub fn maybe_log_level(mut self, level: Option<impl Into<String>>) -> Self {
self.config.log_level = level.map(|l| l.into());
self
}
/// Set request ID headers if Some
pub fn maybe_request_id_headers(mut self, headers: Option<Vec<String>>) -> Self {
self.config.request_id_headers = headers;
self
}
/// Set rate limit tokens per second if Some
pub fn maybe_rate_limit_tokens_per_second(mut self, tokens: Option<i32>) -> Self {
self.config.rate_limit_tokens_per_second = tokens;
self
}
/// Set model path if Some
pub fn maybe_model_path(mut self, path: Option<impl Into<String>>) -> Self {
self.config.model_path = path.map(|p| p.into());
self
}
/// Set tokenizer path if Some
pub fn maybe_tokenizer_path(mut self, path: Option<impl Into<String>>) -> Self {
self.config.tokenizer_path = path.map(|p| p.into());
self
}
/// Set chat template if Some
pub fn maybe_chat_template(mut self, template: Option<impl Into<String>>) -> Self {
self.config.chat_template = template.map(|t| t.into());
self
}
/// Set oracle config if Some
pub fn maybe_oracle(mut self, oracle: Option<OracleConfig>) -> Self {
if let Some(cfg) = oracle {
self.config.history_backend = HistoryBackend::Oracle;
@@ -568,23 +497,19 @@ impl RouterConfigBuilder {
self
}
/// Set reasoning parser if Some
pub fn maybe_reasoning_parser(mut self, parser: Option<impl Into<String>>) -> Self {
self.config.reasoning_parser = parser.map(|p| p.into());
self
}
/// Set tool call parser if Some
pub fn maybe_tool_call_parser(mut self, parser: Option<impl Into<String>>) -> Self {
self.config.tool_call_parser = parser.map(|p| p.into());
self
}
// ==================== mTLS Configuration ====================
// ==================== mTLS ====================
/// Set client certificate and key paths for mTLS authentication
/// Both paths must be provided together
/// Files will be read during build()
/// Both paths must be provided together. Files read during build()
pub fn client_cert_and_key<S1: Into<String>, S2: Into<String>>(
mut self,
cert_path: S1,
@@ -595,8 +520,7 @@ impl RouterConfigBuilder {
self
}
/// Set client certificate and key paths for mTLS if both paths are provided
/// Files will be read during build()
/// Files read during build()
pub fn maybe_client_cert_and_key(
mut self,
cert_path: Option<impl Into<String>>,
@@ -607,49 +531,43 @@ impl RouterConfigBuilder {
self
}
/// Add a CA certificate path for verifying worker TLS certificates
/// File will be read during build()
/// File read during build()
pub fn add_ca_certificate<S: Into<String>>(mut self, ca_cert_path: S) -> Self {
self.ca_cert_paths.push(ca_cert_path.into());
self
}
/// Add multiple CA certificate paths for verifying worker TLS certificates
/// Files will be read during build()
/// Files read during build()
pub fn add_ca_certificates<S: Into<String>>(mut self, ca_cert_paths: Vec<S>) -> Self {
self.ca_cert_paths
.extend(ca_cert_paths.into_iter().map(|p| p.into()));
self
}
// ==================== MCP Configuration ====================
// ==================== MCP ====================
/// Set MCP server configuration file path
/// The config file will be loaded during build()
/// Config file loaded during build()
pub fn mcp_config_path<S: Into<String>>(mut self, path: S) -> Self {
self.mcp_config_path = Some(path.into());
self
}
/// Set MCP server configuration file path if Some
/// Config file loaded during build()
pub fn maybe_mcp_config_path(mut self, path: Option<impl Into<String>>) -> Self {
self.mcp_config_path = path.map(|p| p.into());
self
}
// ==================== Builder Methods ====================
// ==================== Build ====================
/// Build the RouterConfig, validating if requested
pub fn build(self) -> ConfigResult<RouterConfig> {
self.build_with_validation(true)
}
/// Build the RouterConfig without validation
pub fn build_unchecked(self) -> RouterConfig {
self.into()
}
/// Build with optional validation
pub fn build_with_validation(mut self, validate: bool) -> ConfigResult<RouterConfig> {
// Read mTLS certificates from paths if provided
self = self.read_mtls_certificates()?;

2
sgl-router/src/config/mod.rs
View File

@@ -6,7 +6,6 @@ pub use builder::*;
pub use types::*;
pub use validation::*;
/// Configuration errors
#[derive(Debug, thiserror::Error)]
pub enum ConfigError {
#[error("Validation failed: {reason}")]
@@ -26,5 +25,4 @@ pub enum ConfigError {
MissingRequired { field: String },
}
/// Result type for configuration operations
pub type ConfigResult<T> = Result<T, ConfigError>;

126
sgl-router/src/config/types.rs
View File

@@ -8,93 +8,64 @@ use crate::core::ConnectionMode;
/// Main router configuration
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct RouterConfig {
/// Routing mode configuration
pub mode: RoutingMode,
/// Worker connection mode
#[serde(default)]
pub connection_mode: ConnectionMode,
/// Policy configuration
pub policy: PolicyConfig,
/// Server host address
pub host: String,
/// Server port
pub port: u16,
/// Maximum payload size in bytes
pub max_payload_size: usize,
/// Request timeout in seconds
pub request_timeout_secs: u64,
/// Worker startup timeout in seconds
pub worker_startup_timeout_secs: u64,
/// Worker health check interval in seconds
pub worker_startup_check_interval_secs: u64,
/// Enable data parallelism aware schedule
pub dp_aware: bool,
/// The api key used for the authorization with the worker
pub api_key: Option<String>,
/// Service discovery configuration (optional)
pub discovery: Option<DiscoveryConfig>,
/// Metrics configuration (optional)
pub metrics: Option<MetricsConfig>,
/// Log directory (None = stdout only)
pub log_dir: Option<String>,
/// Log level (None = info)
pub log_level: Option<String>,
/// Custom request ID headers to check (defaults to common headers)
pub request_id_headers: Option<Vec<String>>,
/// Maximum concurrent requests allowed (for rate limiting). Set to -1 to disable rate limiting.
/// Set to -1 to disable rate limiting
pub max_concurrent_requests: i32,
/// Queue size for pending requests when max concurrent limit reached (0 = no queue, return 429 immediately)
pub queue_size: usize,
/// Maximum time (in seconds) a request can wait in queue before timing out
pub queue_timeout_secs: u64,
/// Token bucket refill rate (tokens per second). If not set, defaults to max_concurrent_requests
/// If not set, defaults to max_concurrent_requests
pub rate_limit_tokens_per_second: Option<i32>,
/// CORS allowed origins
pub cors_allowed_origins: Vec<String>,
/// Retry configuration
pub retry: RetryConfig,
/// Circuit breaker configuration
pub circuit_breaker: CircuitBreakerConfig,
/// Disable retries (overrides retry.max_retries to 1 when true)
/// When true, overrides retry.max_retries to 1
#[serde(default)]
pub disable_retries: bool,
/// Disable circuit breaker (overrides circuit_breaker.failure_threshold to u32::MAX when true)
/// When true, overrides circuit_breaker.failure_threshold to u32::MAX
#[serde(default)]
pub disable_circuit_breaker: bool,
/// Health check configuration
pub health_check: HealthCheckConfig,
/// Enable Inference Gateway mode (false = proxy mode, true = IGW mode)
#[serde(default)]
pub enable_igw: bool,
/// Model path for loading tokenizer (can be a HuggingFace model ID or local path)
/// Can be a HuggingFace model ID or local path
pub model_path: Option<String>,
/// Explicit tokenizer path (overrides model_path tokenizer if provided)
/// Overrides model_path tokenizer if provided
pub tokenizer_path: Option<String>,
/// Chat template path (optional)
pub chat_template: Option<String>,
/// History backend configuration (memory or none, default: memory)
#[serde(default = "default_history_backend")]
pub history_backend: HistoryBackend,
/// Oracle history backend configuration (required when `history_backend` = "oracle")
/// Required when history_backend = "oracle"
#[serde(skip_serializing_if = "Option::is_none")]
pub oracle: Option<OracleConfig>,
/// Parser for reasoning models (e.g., deepseek-r1, qwen3)
/// For reasoning models (e.g., deepseek-r1, qwen3)
pub reasoning_parser: Option<String>,
/// Parser for handling tool-call interactions
/// For tool-call interactions
pub tool_call_parser: Option<String>,
/// Tokenizer cache configuration
#[serde(default)]
pub tokenizer_cache: TokenizerCacheConfig,
/// mTLS client identity (combined certificate + key in PEM format)
/// This is loaded from client_cert_path and client_key_path during config creation
/// Combined certificate + key in PEM format, loaded from client_cert_path and client_key_path during config creation
#[serde(skip)]
pub client_identity: Option<Vec<u8>>,
/// CA certificates for verifying worker TLS certificates (PEM format)
/// Loaded from ca_cert_paths during config creation
/// PEM format, loaded from ca_cert_paths during config creation
#[serde(default)]
pub ca_certificates: Vec<Vec<u8>>,
/// MCP server configuration (loaded from mcp_config_path during config creation)
/// This is loaded from the config file path and stored here for runtime use
/// Loaded from mcp_config_path during config creation
#[serde(skip)]
pub mcp_config: Option<crate::mcp::McpConfig>,
}
@@ -102,16 +73,14 @@ pub struct RouterConfig {
/// Tokenizer cache configuration
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub struct TokenizerCacheConfig {
/// Enable L0 cache (whole-string exact match)
/// Whole-string exact match cache
#[serde(default = "default_enable_l0")]
pub enable_l0: bool,
/// Maximum number of entries in L0 cache
#[serde(default = "default_l0_max_entries")]
pub l0_max_entries: usize,
/// Enable L1 cache (prefix matching at fixed boundaries)
/// Prefix matching at fixed boundaries
#[serde(default = "default_enable_l1")]
pub enable_l1: bool,
/// Maximum memory for L1 cache in bytes
#[serde(default = "default_l1_max_memory")]
pub l1_max_memory: usize,
}
@@ -151,33 +120,25 @@ fn default_history_backend() -> HistoryBackend {
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
#[serde(rename_all = "lowercase")]
pub enum HistoryBackend {
/// In-memory storage (default)
Memory,
/// No history storage
None,
/// Oracle ATP-backed storage
Oracle,
}
/// Oracle history backend configuration
#[derive(Clone, Serialize, Deserialize, PartialEq)]
pub struct OracleConfig {
/// Directory containing the ATP wallet or TLS config files (optional)
/// ATP wallet or TLS config files directory
#[serde(skip_serializing_if = "Option::is_none")]
pub wallet_path: Option<String>,
/// Connection descriptor / DSN (e.g. `tcps://host:port/service`)
/// DSN (e.g. `tcps://host:port/service`)
pub connect_descriptor: String,
/// Database username
pub username: String,
/// Database password
pub password: String,
/// Minimum number of pooled connections to keep ready
#[serde(default = "default_pool_min")]
pub pool_min: usize,
/// Maximum number of pooled connections
#[serde(default = "default_pool_max")]
pub pool_max: usize,
/// Maximum time to wait for a connection from the pool (seconds)
#[serde(default = "default_pool_timeout_secs")]
pub pool_timeout_secs: u64,
}
@@ -226,28 +187,19 @@ impl std::fmt::Debug for OracleConfig {
#[serde(tag = "type")]
pub enum RoutingMode {
#[serde(rename = "regular")]
Regular {
/// List of worker URLs
worker_urls: Vec<String>,
},
Regular { worker_urls: Vec<String> },
#[serde(rename = "prefill_decode")]
PrefillDecode {
/// Prefill worker URLs with optional bootstrap ports
/// With optional bootstrap ports
prefill_urls: Vec<(String, Option<u16>)>,
/// Decode worker URLs
decode_urls: Vec<String>,
/// Optional separate policy for prefill workers
#[serde(skip_serializing_if = "Option::is_none")]
prefill_policy: Option<PolicyConfig>,
/// Optional separate policy for decode workers
#[serde(skip_serializing_if = "Option::is_none")]
decode_policy: Option<PolicyConfig>,
},
#[serde(rename = "openai")]
OpenAI {
/// OpenAI-compatible API base(s), provided via worker URLs
worker_urls: Vec<String>,
},
OpenAI { worker_urls: Vec<String> },
}
impl RoutingMode {
@@ -302,23 +254,15 @@ pub enum PolicyConfig {
#[serde(rename = "cache_aware")]
CacheAware {
/// Minimum prefix match ratio to use cache-based routing
cache_threshold: f32,
/// Absolute load difference threshold for load balancing
balance_abs_threshold: usize,
/// Relative load ratio threshold for load balancing
balance_rel_threshold: f32,
/// Interval between cache eviction cycles (seconds)
eviction_interval_secs: u64,
/// Maximum cache tree size per tenant
max_tree_size: usize,
},
#[serde(rename = "power_of_two")]
PowerOfTwo {
/// Interval for load monitoring (seconds)
load_check_interval_secs: u64,
},
PowerOfTwo { load_check_interval_secs: u64 },
}
impl PolicyConfig {
@@ -335,21 +279,17 @@ impl PolicyConfig {
/// Service discovery configuration
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct DiscoveryConfig {
/// Enable service discovery
pub enabled: bool,
/// Kubernetes namespace (None = all namespaces)
/// None = all namespaces
pub namespace: Option<String>,
/// Service discovery port
pub port: u16,
/// Check interval for service discovery
pub check_interval_secs: u64,
/// Regular mode selector
/// Regular mode
pub selector: HashMap<String, String>,
/// PD mode prefill selector
/// PD mode prefill
pub prefill_selector: HashMap<String, String>,
/// PD mode decode selector
/// PD mode decode
pub decode_selector: HashMap<String, String>,
/// Bootstrap port annotation key
pub bootstrap_port_annotation: String,
}
@@ -371,16 +311,11 @@ impl Default for DiscoveryConfig {
/// Retry configuration for request handling
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct RetryConfig {
/// Maximum number of retry attempts
pub max_retries: u32,
/// Initial backoff delay in milliseconds
pub initial_backoff_ms: u64,
/// Maximum backoff delay in milliseconds
pub max_backoff_ms: u64,
/// Backoff multiplier for exponential backoff
pub backoff_multiplier: f32,
/// Jitter factor applied to backoff (0.0 - 1.0)
/// Effective delay D' = D * (1 + U[-j, +j])
/// D' = D * (1 + U[-j, +j]) where j is jitter factor
#[serde(default = "default_retry_jitter_factor")]
pub jitter_factor: f32,
}
@@ -404,15 +339,10 @@ fn default_retry_jitter_factor() -> f32 {
/// Health check configuration for worker monitoring
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct HealthCheckConfig {
/// Number of consecutive failures before marking unhealthy
pub failure_threshold: u32,
/// Number of consecutive successes before marking healthy
pub success_threshold: u32,
/// Timeout for health check requests in seconds
pub timeout_secs: u64,
/// Interval between health checks in seconds
pub check_interval_secs: u64,
/// Health check endpoint path
pub endpoint: String,
}
@@ -431,13 +361,9 @@ impl Default for HealthCheckConfig {
/// Circuit breaker configuration for worker reliability
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct CircuitBreakerConfig {
/// Number of consecutive failures before opening circuit
pub failure_threshold: u32,
/// Number of consecutive successes before closing circuit
pub success_threshold: u32,
/// Time before attempting to recover from open state (in seconds)
pub timeout_duration_secs: u64,
/// Window duration for failure tracking (in seconds)
pub window_duration_secs: u64,
}
@@ -455,9 +381,7 @@ impl Default for CircuitBreakerConfig {
/// Metrics configuration
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct MetricsConfig {
/// Prometheus metrics port
pub port: u16,
/// Prometheus metrics host
pub host: String,
}

63
sgl-router/src/config/validation.rs
View File

@@ -5,9 +5,7 @@ use crate::core::ConnectionMode;
pub struct ConfigValidator;
impl ConfigValidator {
/// Validate a complete router configuration
pub fn validate(config: &RouterConfig) -> ConfigResult<()> {
// Check if service discovery is enabled
let has_service_discovery = config.discovery.as_ref().is_some_and(|d| d.enabled);
Self::validate_mode(&config.mode, has_service_discovery)?;
@@ -24,55 +22,46 @@ impl ConfigValidator {
Self::validate_compatibility(config)?;
// Validate effective retry/CB configs (respect disable flags)
let retry_cfg = config.effective_retry_config();
let cb_cfg = config.effective_circuit_breaker_config();
Self::validate_retry(&retry_cfg)?;
Self::validate_circuit_breaker(&cb_cfg)?;
// Validate Oracle configuration if enabled
if config.history_backend == HistoryBackend::Oracle {
if config.oracle.is_none() {
return Err(ConfigError::MissingRequired {
field: "oracle".to_string(),
});
}
// Validate Oracle configuration details
if let Some(oracle) = &config.oracle {
Self::validate_oracle(oracle)?;
}
}
// Validate tokenizer cache configuration
Self::validate_tokenizer_cache(&config.tokenizer_cache)?;
Ok(())
}
/// Validate Oracle configuration
fn validate_oracle(oracle: &OracleConfig) -> ConfigResult<()> {
// Validate username is not empty
if oracle.username.is_empty() {
return Err(ConfigError::MissingRequired {
field: "oracle.username".to_string(),
});
}
// Validate password is not empty
if oracle.password.is_empty() {
return Err(ConfigError::MissingRequired {
field: "oracle.password".to_string(),
});
}
// Validate connect_descriptor is not empty
if oracle.connect_descriptor.is_empty() {
return Err(ConfigError::MissingRequired {
field: "oracle_dsn or oracle_tns_alias".to_string(),
});
}
// Validate pool_min is at least 1
if oracle.pool_min < 1 {
return Err(ConfigError::InvalidValue {
field: "oracle.pool_min".to_string(),
@@ -81,7 +70,6 @@ impl ConfigValidator {
});
}
// Validate pool_max is greater than or equal to pool_min
if oracle.pool_max < oracle.pool_min {
return Err(ConfigError::InvalidValue {
field: "oracle.pool_max".to_string(),
@@ -90,7 +78,6 @@ impl ConfigValidator {
});
}
// Validate pool_timeout_secs is greater than 0
if oracle.pool_timeout_secs == 0 {
return Err(ConfigError::InvalidValue {
field: "oracle.pool_timeout_secs".to_string(),
@@ -102,17 +89,13 @@ impl ConfigValidator {
Ok(())
}
/// Validate routing mode configuration
fn validate_mode(mode: &RoutingMode, has_service_discovery: bool) -> ConfigResult<()> {
match mode {
RoutingMode::Regular { worker_urls } => {
// Validate URLs if any are provided
if !worker_urls.is_empty() {
Self::validate_urls(worker_urls)?;
}
// Note: We allow empty worker URLs even without service discovery
// to let the router start and fail at runtime when routing requests.
// This matches legacy behavior and test expectations.
// Allow empty URLs without service discovery to match legacy behavior
}
RoutingMode::PrefillDecode {
prefill_urls,
@@ -120,7 +103,6 @@ impl ConfigValidator {
prefill_policy,
decode_policy,
} => {
// Only require URLs if service discovery is disabled
if !has_service_discovery {
if prefill_urls.is_empty() {
return Err(ConfigError::ValidationFailed {
@@ -134,7 +116,6 @@ impl ConfigValidator {
}
}
// Validate URLs if any are provided
if !prefill_urls.is_empty() {
let prefill_url_strings: Vec<String> =
prefill_urls.iter().map(|(url, _)| url.clone()).collect();
@@ -144,7 +125,6 @@ impl ConfigValidator {
Self::validate_urls(decode_urls)?;
}
// Validate bootstrap ports
for (_url, port) in prefill_urls {
if let Some(port) = port {
if *port == 0 {
@@ -157,7 +137,6 @@ impl ConfigValidator {
}
}
// Validate optional prefill and decode policies
if let Some(p_policy) = prefill_policy {
Self::validate_policy(p_policy)?;
}
@@ -166,25 +145,20 @@ impl ConfigValidator {
}
}
RoutingMode::OpenAI { worker_urls } => {
// Require at least one worker URL for OpenAI router
if worker_urls.is_empty() {
return Err(ConfigError::ValidationFailed {
reason: "OpenAI mode requires at least one --worker-urls entry".to_string(),
});
}
// Validate URLs
Self::validate_urls(worker_urls)?;
}
}
Ok(())
}
/// Validate policy configuration
fn validate_policy(policy: &PolicyConfig) -> ConfigResult<()> {
match policy {
PolicyConfig::Random | PolicyConfig::RoundRobin => {
// No specific validation needed
}
PolicyConfig::Random | PolicyConfig::RoundRobin => {}
PolicyConfig::CacheAware {
cache_threshold,
balance_abs_threshold: _,
@@ -239,7 +213,6 @@ impl ConfigValidator {
Ok(())
}
/// Validate server configuration
fn validate_server_settings(config: &RouterConfig) -> ConfigResult<()> {
if config.port == 0 {
return Err(ConfigError::InvalidValue {
@@ -302,10 +275,9 @@ impl ConfigValidator {
Ok(())
}
/// Validate service discovery configuration
fn validate_discovery(discovery: &DiscoveryConfig, mode: &RoutingMode) -> ConfigResult<()> {
if !discovery.enabled {
return Ok(()); // No validation needed if disabled
return Ok(());
}
if discovery.port == 0 {
@@ -324,7 +296,6 @@ impl ConfigValidator {
});
}
// Validate selectors based on mode
match mode {
RoutingMode::Regular { .. } => {
if discovery.selector.is_empty() {
@@ -342,7 +313,6 @@ impl ConfigValidator {
}
}
RoutingMode::OpenAI { .. } => {
// OpenAI mode doesn't use service discovery
return Err(ConfigError::ValidationFailed {
reason: "OpenAI mode does not support service discovery".to_string(),
});
@@ -352,7 +322,6 @@ impl ConfigValidator {
Ok(())
}
/// Validate metrics configuration
fn validate_metrics(metrics: &MetricsConfig) -> ConfigResult<()> {
if metrics.port == 0 {
return Err(ConfigError::InvalidValue {
@@ -373,7 +342,6 @@ impl ConfigValidator {
Ok(())
}
/// Validate retry configuration
fn validate_retry(retry: &RetryConfig) -> ConfigResult<()> {
if retry.max_retries < 1 {
return Err(ConfigError::InvalidValue {
@@ -413,7 +381,6 @@ impl ConfigValidator {
Ok(())
}
/// Validate circuit breaker configuration
fn validate_circuit_breaker(cb: &CircuitBreakerConfig) -> ConfigResult<()> {
if cb.failure_threshold < 1 {
return Err(ConfigError::InvalidValue {
@@ -446,9 +413,7 @@ impl ConfigValidator {
Ok(())
}
/// Validate tokenizer cache configuration
fn validate_tokenizer_cache(cache: &TokenizerCacheConfig) -> ConfigResult<()> {
// Validate L0 max entries when L0 is enabled
if cache.enable_l0 && cache.l0_max_entries == 0 {
return Err(ConfigError::InvalidValue {
field: "tokenizer_cache.l0_max_entries".to_string(),
@@ -457,7 +422,6 @@ impl ConfigValidator {
});
}
// Validate L1 max memory when L1 is enabled
if cache.enable_l1 && cache.l1_max_memory == 0 {
return Err(ConfigError::InvalidValue {
field: "tokenizer_cache.l1_max_memory".to_string(),
@@ -469,9 +433,7 @@ impl ConfigValidator {
Ok(())
}
/// Validate mTLS certificate configuration
fn validate_mtls(config: &RouterConfig) -> ConfigResult<()> {
// Validate that if we have client_identity, it's not empty
if let Some(identity) = &config.client_identity {
if identity.is_empty() {
return Err(ConfigError::ValidationFailed {
@@ -480,7 +442,6 @@ impl ConfigValidator {
}
}
// Validate CA certificates are not empty
for (idx, ca_cert) in config.ca_certificates.iter().enumerate() {
if ca_cert.is_empty() {
return Err(ConfigError::ValidationFailed {
@@ -492,14 +453,11 @@ impl ConfigValidator {
Ok(())
}
/// Validate compatibility between different configuration sections
fn validate_compatibility(config: &RouterConfig) -> ConfigResult<()> {
// IGW mode is independent - skip other compatibility checks when enabled
if config.enable_igw {
return Ok(());
}
// Validate gRPC connection mode requires tokenizer configuration
if matches!(config.connection_mode, ConnectionMode::Grpc { .. })
&& config.tokenizer_path.is_none()
&& config.model_path.is_none()
@@ -509,18 +467,11 @@ impl ConfigValidator {
});
}
// Validate mTLS configuration
Self::validate_mtls(config)?;
// All policies are now supported for both router types thanks to the unified trait design
// No mode/policy restrictions needed anymore
// Check if service discovery is enabled for worker count validation
let has_service_discovery = config.discovery.as_ref().is_some_and(|d| d.enabled);
// Only validate worker counts if service discovery is disabled
if !has_service_discovery {
// Check if power-of-two policy makes sense with insufficient workers
if let PolicyConfig::PowerOfTwo { .. } = &config.policy {
let worker_count = config.mode.worker_count();
if worker_count < 2 {
@@ -530,7 +481,6 @@ impl ConfigValidator {
}
}
// For PD mode, validate that policies have sufficient workers
if let RoutingMode::PrefillDecode {
prefill_urls,
decode_urls,
@@ -538,7 +488,6 @@ impl ConfigValidator {
decode_policy,
} = &config.mode
{
// Check power-of-two for prefill
if let Some(PolicyConfig::PowerOfTwo { .. }) = prefill_policy {
if prefill_urls.len() < 2 {
return Err(ConfigError::IncompatibleConfig {
@@ -547,7 +496,6 @@ impl ConfigValidator {
}
}
// Check power-of-two for decode
if let Some(PolicyConfig::PowerOfTwo { .. }) = decode_policy {
if decode_urls.len() < 2 {
return Err(ConfigError::IncompatibleConfig {
@@ -560,8 +508,6 @@ impl ConfigValidator {
}
}
// Service discovery is conflict with dp_aware routing for now
// since it's not fully supported yet
if has_service_discovery && config.dp_aware {
return Err(ConfigError::IncompatibleConfig {
reason: "DP-aware routing is not compatible with service discovery".to_string(),
@@ -571,7 +517,6 @@ impl ConfigValidator {
Ok(())
}
/// Validate URL format
fn validate_urls(urls: &[String]) -> ConfigResult<()> {
for url in urls {
if url.is_empty() {
@@ -593,10 +538,8 @@ impl ConfigValidator {
});
}
// Basic URL validation
match ::url::Url::parse(url) {
Ok(parsed) => {
// Additional validation
if parsed.host_str().is_none() {
return Err(ConfigError::InvalidValue {
field: "worker_url".to_string(),