From 9188fecccad3c4b1c5fc9d063f8d1b49eb214936 Mon Sep 17 00:00:00 2001 From: Simo Lin Date: Mon, 17 Nov 2025 19:28:21 -0800 Subject: [PATCH] [model-gateway] use worker startup time out for worker registration (#13473) --- sgl-router/bindings/python/sglang_router/router.py | 2 +- sgl-router/bindings/python/sglang_router/router_args.py | 4 ++-- sgl-router/src/config/types.rs | 8 ++++---- sgl-router/src/core/workflow/steps/worker_registration.rs | 7 ++++--- sgl-router/src/main.rs | 2 +- 5 files changed, 12 insertions(+), 11 deletions(-) diff --git a/sgl-router/bindings/python/sglang_router/router.py b/sgl-router/bindings/python/sglang_router/router.py index 0ee130749..22e4d4a38 100644 --- a/sgl-router/bindings/python/sglang_router/router.py +++ b/sgl-router/bindings/python/sglang_router/router.py @@ -74,7 +74,7 @@ class Router: - PolicyType.PowerOfTwo: Select best of two random workers based on load (PD mode only) host: Host address to bind the router server. Supports IPv4, IPv6 (e.g., ::, ::1), or 0.0.0.0 for all interfaces. Default: '0.0.0.0' port: Port number to bind the router server. Default: 3001 - worker_startup_timeout_secs: Timeout in seconds for worker startup. Default: 300 + worker_startup_timeout_secs: Timeout in seconds for worker startup and registration. Large models can take significant time to load into GPU memory. Default: 1800 (30 minutes) worker_startup_check_interval: Interval in seconds between checks for worker initialization. Default: 10 cache_threshold: Cache threshold (0.0-1.0) for cache-aware routing. Routes to cached worker if the match rate exceeds threshold, otherwise routes to the worker with the smallest diff --git a/sgl-router/bindings/python/sglang_router/router_args.py b/sgl-router/bindings/python/sglang_router/router_args.py index 04077b9de..fabc65756 100644 --- a/sgl-router/bindings/python/sglang_router/router_args.py +++ b/sgl-router/bindings/python/sglang_router/router_args.py @@ -26,7 +26,7 @@ class RouterArgs: policy: str = "cache_aware" prefill_policy: Optional[str] = None # Specific policy for prefill nodes in PD mode decode_policy: Optional[str] = None # Specific policy for decode nodes in PD mode - worker_startup_timeout_secs: int = 600 + worker_startup_timeout_secs: int = 1800 worker_startup_check_interval: int = 30 cache_threshold: float = 0.3 balance_abs_threshold: int = 64 @@ -209,7 +209,7 @@ class RouterArgs: f"--{prefix}worker-startup-timeout-secs", type=int, default=RouterArgs.worker_startup_timeout_secs, - help="Timeout in seconds for worker startup", + help="Timeout in seconds for worker startup and registration (default: 1800 / 30 minutes). Large models can take significant time to load into GPU memory.", ) parser.add_argument( f"--{prefix}worker-startup-check-interval", diff --git a/sgl-router/src/config/types.rs b/sgl-router/src/config/types.rs index 90174f070..6bfd53cff 100644 --- a/sgl-router/src/config/types.rs +++ b/sgl-router/src/config/types.rs @@ -467,9 +467,9 @@ impl Default for RouterConfig { policy: PolicyConfig::Random, host: "0.0.0.0".to_string(), port: 3001, - max_payload_size: 536_870_912, // 512MB - request_timeout_secs: 1800, // 30 minutes - worker_startup_timeout_secs: 600, + max_payload_size: 536_870_912, // 512MB + request_timeout_secs: 1800, // 30 minutes + worker_startup_timeout_secs: 1800, // 30 minutes for large model loading worker_startup_check_interval_secs: 30, dp_aware: false, api_key: None, @@ -580,7 +580,7 @@ mod tests { assert_eq!(config.port, 3001); assert_eq!(config.max_payload_size, 536_870_912); assert_eq!(config.request_timeout_secs, 1800); - assert_eq!(config.worker_startup_timeout_secs, 600); + assert_eq!(config.worker_startup_timeout_secs, 1800); assert_eq!(config.worker_startup_check_interval_secs, 30); assert!(config.discovery.is_none()); assert!(config.metrics.is_none()); diff --git a/sgl-router/src/core/workflow/steps/worker_registration.rs b/sgl-router/src/core/workflow/steps/worker_registration.rs index 795953a4b..af21c30e2 100644 --- a/sgl-router/src/core/workflow/steps/worker_registration.rs +++ b/sgl-router/src/core/workflow/steps/worker_registration.rs @@ -898,10 +898,11 @@ impl StepExecutor for ActivateWorkerStep { pub fn create_worker_registration_workflow( router_config: &crate::config::RouterConfig, ) -> WorkflowDefinition { - // Use health check timeout from config with 30 second buffer as workflow-level upper bound - let detect_timeout = Duration::from_secs(router_config.health_check.timeout_secs + 30); + // Use startup timeout from config for worker registration + // This is separate from health_check.timeout_secs which is for individual HTTP requests + let detect_timeout = Duration::from_secs(router_config.worker_startup_timeout_secs); - // Calculate max_attempts to match the detect_timeout + // Calculate max_attempts to match the startup_timeout // With Linear backoff (increment 1s, max 5s): // - Attempts 1-5: 0s, 1s, 2s, 3s, 4s = 10s total // - Attempts 6+: 5s each diff --git a/sgl-router/src/main.rs b/sgl-router/src/main.rs index 34c9c4609..b8e7839be 100644 --- a/sgl-router/src/main.rs +++ b/sgl-router/src/main.rs @@ -126,7 +126,7 @@ struct CliArgs { #[arg(long, value_parser = ["random", "round_robin", "cache_aware", "power_of_two"])] decode_policy: Option, - #[arg(long, default_value_t = 600)] + #[arg(long, default_value_t = 1800)] worker_startup_timeout_secs: u64, #[arg(long, default_value_t = 30)]