[model-gateway] fix graceful shutdown for TLS/Non-TLS server (#15491)

sgl-model-gateway/bindings/python/sglang_router/router_args.py

@@ -59,6 +59,8 @@ class RouterArgs:
     request_id_headers: Optional[List[str]] = None
     # Request timeout in seconds
     request_timeout_secs: int = 1800
+    # Grace period in seconds to wait for in-flight requests during shutdown
+    shutdown_grace_period_secs: int = 180
     # Max concurrent requests for rate limiting (-1 to disable)
     max_concurrent_requests: int = -1
     # Queue size for pending requests when max concurrent limit reached
@@ -364,6 +366,12 @@ class RouterArgs:
             default=RouterArgs.request_timeout_secs,
             help="Request timeout in seconds",
         )
+        parser.add_argument(
+            f"--{prefix}shutdown-grace-period-secs",
+            type=int,
+            default=RouterArgs.shutdown_grace_period_secs,
+            help="Grace period in seconds to wait for in-flight requests during shutdown",
+        )
         # Retry configuration
         parser.add_argument(
             f"--{prefix}retry-max-retries",

sgl-model-gateway/bindings/python/src/lib.rs

@@ -179,6 +179,7 @@ struct Router {
     prometheus_host: Option<String>,
     prometheus_duration_buckets: Option<Vec<f64>>,
     request_timeout_secs: u64,
+    shutdown_grace_period_secs: u64,
     request_id_headers: Option<Vec<String>>,
     pd_disaggregation: bool,
     bucket_adjust_interval_secs: usize,
@@ -448,6 +449,7 @@ impl Router {
         prometheus_host = None,
         prometheus_duration_buckets = None,
         request_timeout_secs = 1800,
+        shutdown_grace_period_secs = 180,
         request_id_headers = None,
         pd_disaggregation = false,
         bucket_adjust_interval_secs = 5,
@@ -528,6 +530,7 @@ impl Router {
         prometheus_host: Option<String>,
         prometheus_duration_buckets: Option<Vec<f64>>,
         request_timeout_secs: u64,
+        shutdown_grace_period_secs: u64,
         request_id_headers: Option<Vec<String>>,
         pd_disaggregation: bool,
         bucket_adjust_interval_secs: usize,
@@ -621,6 +624,7 @@ impl Router {
             prometheus_host,
             prometheus_duration_buckets,
             request_timeout_secs,
+            shutdown_grace_period_secs,
             request_id_headers,
             pd_disaggregation,
             bucket_adjust_interval_secs,
@@ -729,6 +733,7 @@ impl Router {
                 prometheus_config,
                 request_timeout_secs: self.request_timeout_secs,
                 request_id_headers: self.request_id_headers.clone(),
+                shutdown_grace_period_secs: self.shutdown_grace_period_secs,
             })
             .await
             .map_err(|e| pyo3::exceptions::PyRuntimeError::new_err(e.to_string()))