Change routing policy API to be async to support more policies (#17048)

sgl-model-gateway/src/routers/http/pd_router.rs

@@ -747,7 +747,8 @@ impl PDRouter {
             headers,
             hash_ring.clone(),
             "prefill",
-        )?;
+        )
+        .await?;
 
         let decode = Self::pick_worker_by_policy_arc(
             &decode_workers,
@@ -756,7 +757,8 @@ impl PDRouter {
             headers,
             hash_ring,
             "decode",
-        )?;
+        )
+        .await?;
 
         // Record worker selection metrics (Layer 3)
         let model = model_id.unwrap_or(UNKNOWN_MODEL_ID);
@@ -776,7 +778,7 @@ impl PDRouter {
         Ok((prefill, decode))
     }
 
-    fn pick_worker_by_policy_arc(
+    async fn pick_worker_by_policy_arc(
         workers: &[Arc<dyn Worker>],
         policy: &dyn LoadBalancingPolicy,
         request_text: Option<&str>,
@@ -814,6 +816,7 @@ impl PDRouter {
                     hash_ring,
                 },
             )
+            .await
             .ok_or_else(|| {
                 format!(
                     "Policy {} failed to select a {} worker",

sgl-model-gateway/src/routers/http/router.rs

@@ -130,7 +130,7 @@ impl Router {
     }
 
     /// Select worker for a specific model considering circuit breaker state
-    fn select_worker_for_model(
+    async fn select_worker_for_model(
         &self,
         model_id: Option<&str>,
         text: Option<&str>,
@@ -167,15 +167,17 @@ impl Router {
             .worker_registry
             .get_hash_ring(effective_model_id.unwrap_or(UNKNOWN_MODEL_ID));
 
-        let idx = policy.select_worker(
-            &available,
-            &SelectWorkerInfo {
-                request_text: text,
-                tokens: None, // HTTP doesn't have tokens, use gRPC for PrefixHash
-                headers,
-                hash_ring,
-            },
-        )?;
+        let idx = policy
+            .select_worker(
+                &available,
+                &SelectWorkerInfo {
+                    request_text: text,
+                    tokens: None, // HTTP doesn't have tokens, use gRPC for PrefixHash
+                    headers,
+                    hash_ring,
+                },
+            )
+            .await?;
 
         // Record worker selection metric (Layer 3)
         Metrics::record_worker_selection(
@@ -276,7 +278,10 @@ impl Router {
         is_stream: bool,
         text: &str,
     ) -> Response {
-        let worker = match self.select_worker_for_model(model_id, Some(text), headers) {
+        let worker = match self
+            .select_worker_for_model(model_id, Some(text), headers)
+            .await
+        {
             Some(w) => w,
             None => {
                 return error::service_unavailable(