Change routing policy API to be async to support more policies (#17048)

This commit is contained in:
fzyzcjy
2026-01-17 17:12:51 +08:00
committed by GitHub
parent d2c863878c
commit 9c2530642c
16 changed files with 350 additions and 217 deletions
@@ -78,12 +78,10 @@ impl PipelineStage for WorkerSelectionStage {
let workers = match self.mode {
WorkerSelectionMode::Regular => {
match self.select_single_worker(
ctx.input.model_id.as_deref(),
text,
tokens,
headers,
) {
match self
.select_single_worker(ctx.input.model_id.as_deref(), text, tokens, headers)
.await
{
Some(w) => WorkerSelection::Single { worker: w },
None => {
let model = ctx.input.model_id.as_deref().unwrap_or(UNKNOWN_MODEL_ID);
@@ -101,7 +99,10 @@ impl PipelineStage for WorkerSelectionStage {
}
}
WorkerSelectionMode::PrefillDecode => {
match self.select_pd_pair(ctx.input.model_id.as_deref(), text, tokens, headers) {
match self
.select_pd_pair(ctx.input.model_id.as_deref(), text, tokens, headers)
.await
{
Some((prefill, decode)) => WorkerSelection::Dual { prefill, decode },
None => {
let model = ctx.input.model_id.as_deref().unwrap_or(UNKNOWN_MODEL_ID);
@@ -130,7 +131,7 @@ impl PipelineStage for WorkerSelectionStage {
}
impl WorkerSelectionStage {
fn select_single_worker(
async fn select_single_worker(
&self,
model_id: Option<&str>,
text: Option<&str>,
@@ -166,15 +167,17 @@ impl WorkerSelectionStage {
.get_hash_ring(model_id.unwrap_or(UNKNOWN_MODEL_ID));
// Select worker using the policy
let idx = policy.select_worker(
&available,
&SelectWorkerInfo {
request_text: text,
tokens,
headers,
hash_ring,
},
)?;
let idx = policy
.select_worker(
&available,
&SelectWorkerInfo {
request_text: text,
tokens,
headers,
hash_ring,
},
)
.await?;
let selected = available[idx].clone();
// Record worker selection metric
@@ -188,7 +191,7 @@ impl WorkerSelectionStage {
Some(selected)
}
fn select_pd_pair(
async fn select_pd_pair(
&self,
model_id: Option<&str>,
text: Option<&str>,
@@ -244,8 +247,8 @@ impl WorkerSelectionStage {
headers,
hash_ring,
};
let prefill_idx = policy.select_worker(&available_prefill, &info)?;
let decode_idx = policy.select_worker(&available_decode, &info)?;
let prefill_idx = policy.select_worker(&available_prefill, &info).await?;
let decode_idx = policy.select_worker(&available_decode, &info).await?;
let model = model_id.unwrap_or(UNKNOWN_MODEL_ID);
let policy_name = policy.name();
@@ -747,7 +747,8 @@ impl PDRouter {
headers,
hash_ring.clone(),
"prefill",
)?;
)
.await?;
let decode = Self::pick_worker_by_policy_arc(
&decode_workers,
@@ -756,7 +757,8 @@ impl PDRouter {
headers,
hash_ring,
"decode",
)?;
)
.await?;
// Record worker selection metrics (Layer 3)
let model = model_id.unwrap_or(UNKNOWN_MODEL_ID);
@@ -776,7 +778,7 @@ impl PDRouter {
Ok((prefill, decode))
}
fn pick_worker_by_policy_arc(
async fn pick_worker_by_policy_arc(
workers: &[Arc<dyn Worker>],
policy: &dyn LoadBalancingPolicy,
request_text: Option<&str>,
@@ -814,6 +816,7 @@ impl PDRouter {
hash_ring,
},
)
.await
.ok_or_else(|| {
format!(
"Policy {} failed to select a {} worker",
+16 -11
View File
@@ -130,7 +130,7 @@ impl Router {
}
/// Select worker for a specific model considering circuit breaker state
fn select_worker_for_model(
async fn select_worker_for_model(
&self,
model_id: Option<&str>,
text: Option<&str>,
@@ -167,15 +167,17 @@ impl Router {
.worker_registry
.get_hash_ring(effective_model_id.unwrap_or(UNKNOWN_MODEL_ID));
let idx = policy.select_worker(
&available,
&SelectWorkerInfo {
request_text: text,
tokens: None, // HTTP doesn't have tokens, use gRPC for PrefixHash
headers,
hash_ring,
},
)?;
let idx = policy
.select_worker(
&available,
&SelectWorkerInfo {
request_text: text,
tokens: None, // HTTP doesn't have tokens, use gRPC for PrefixHash
headers,
hash_ring,
},
)
.await?;
// Record worker selection metric (Layer 3)
Metrics::record_worker_selection(
@@ -276,7 +278,10 @@ impl Router {
is_stream: bool,
text: &str,
) -> Response {
let worker = match self.select_worker_for_model(model_id, Some(text), headers) {
let worker = match self
.select_worker_for_model(model_id, Some(text), headers)
.await
{
Some(w) => w,
None => {
return error::service_unavailable(