Change routing policy API to be async to support more policies (#17048)
This commit is contained in:
@@ -78,12 +78,10 @@ impl PipelineStage for WorkerSelectionStage {
|
||||
|
||||
let workers = match self.mode {
|
||||
WorkerSelectionMode::Regular => {
|
||||
match self.select_single_worker(
|
||||
ctx.input.model_id.as_deref(),
|
||||
text,
|
||||
tokens,
|
||||
headers,
|
||||
) {
|
||||
match self
|
||||
.select_single_worker(ctx.input.model_id.as_deref(), text, tokens, headers)
|
||||
.await
|
||||
{
|
||||
Some(w) => WorkerSelection::Single { worker: w },
|
||||
None => {
|
||||
let model = ctx.input.model_id.as_deref().unwrap_or(UNKNOWN_MODEL_ID);
|
||||
@@ -101,7 +99,10 @@ impl PipelineStage for WorkerSelectionStage {
|
||||
}
|
||||
}
|
||||
WorkerSelectionMode::PrefillDecode => {
|
||||
match self.select_pd_pair(ctx.input.model_id.as_deref(), text, tokens, headers) {
|
||||
match self
|
||||
.select_pd_pair(ctx.input.model_id.as_deref(), text, tokens, headers)
|
||||
.await
|
||||
{
|
||||
Some((prefill, decode)) => WorkerSelection::Dual { prefill, decode },
|
||||
None => {
|
||||
let model = ctx.input.model_id.as_deref().unwrap_or(UNKNOWN_MODEL_ID);
|
||||
@@ -130,7 +131,7 @@ impl PipelineStage for WorkerSelectionStage {
|
||||
}
|
||||
|
||||
impl WorkerSelectionStage {
|
||||
fn select_single_worker(
|
||||
async fn select_single_worker(
|
||||
&self,
|
||||
model_id: Option<&str>,
|
||||
text: Option<&str>,
|
||||
@@ -166,15 +167,17 @@ impl WorkerSelectionStage {
|
||||
.get_hash_ring(model_id.unwrap_or(UNKNOWN_MODEL_ID));
|
||||
|
||||
// Select worker using the policy
|
||||
let idx = policy.select_worker(
|
||||
&available,
|
||||
&SelectWorkerInfo {
|
||||
request_text: text,
|
||||
tokens,
|
||||
headers,
|
||||
hash_ring,
|
||||
},
|
||||
)?;
|
||||
let idx = policy
|
||||
.select_worker(
|
||||
&available,
|
||||
&SelectWorkerInfo {
|
||||
request_text: text,
|
||||
tokens,
|
||||
headers,
|
||||
hash_ring,
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
let selected = available[idx].clone();
|
||||
|
||||
// Record worker selection metric
|
||||
@@ -188,7 +191,7 @@ impl WorkerSelectionStage {
|
||||
Some(selected)
|
||||
}
|
||||
|
||||
fn select_pd_pair(
|
||||
async fn select_pd_pair(
|
||||
&self,
|
||||
model_id: Option<&str>,
|
||||
text: Option<&str>,
|
||||
@@ -244,8 +247,8 @@ impl WorkerSelectionStage {
|
||||
headers,
|
||||
hash_ring,
|
||||
};
|
||||
let prefill_idx = policy.select_worker(&available_prefill, &info)?;
|
||||
let decode_idx = policy.select_worker(&available_decode, &info)?;
|
||||
let prefill_idx = policy.select_worker(&available_prefill, &info).await?;
|
||||
let decode_idx = policy.select_worker(&available_decode, &info).await?;
|
||||
|
||||
let model = model_id.unwrap_or(UNKNOWN_MODEL_ID);
|
||||
let policy_name = policy.name();
|
||||
|
||||
@@ -747,7 +747,8 @@ impl PDRouter {
|
||||
headers,
|
||||
hash_ring.clone(),
|
||||
"prefill",
|
||||
)?;
|
||||
)
|
||||
.await?;
|
||||
|
||||
let decode = Self::pick_worker_by_policy_arc(
|
||||
&decode_workers,
|
||||
@@ -756,7 +757,8 @@ impl PDRouter {
|
||||
headers,
|
||||
hash_ring,
|
||||
"decode",
|
||||
)?;
|
||||
)
|
||||
.await?;
|
||||
|
||||
// Record worker selection metrics (Layer 3)
|
||||
let model = model_id.unwrap_or(UNKNOWN_MODEL_ID);
|
||||
@@ -776,7 +778,7 @@ impl PDRouter {
|
||||
Ok((prefill, decode))
|
||||
}
|
||||
|
||||
fn pick_worker_by_policy_arc(
|
||||
async fn pick_worker_by_policy_arc(
|
||||
workers: &[Arc<dyn Worker>],
|
||||
policy: &dyn LoadBalancingPolicy,
|
||||
request_text: Option<&str>,
|
||||
@@ -814,6 +816,7 @@ impl PDRouter {
|
||||
hash_ring,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.ok_or_else(|| {
|
||||
format!(
|
||||
"Policy {} failed to select a {} worker",
|
||||
|
||||
@@ -130,7 +130,7 @@ impl Router {
|
||||
}
|
||||
|
||||
/// Select worker for a specific model considering circuit breaker state
|
||||
fn select_worker_for_model(
|
||||
async fn select_worker_for_model(
|
||||
&self,
|
||||
model_id: Option<&str>,
|
||||
text: Option<&str>,
|
||||
@@ -167,15 +167,17 @@ impl Router {
|
||||
.worker_registry
|
||||
.get_hash_ring(effective_model_id.unwrap_or(UNKNOWN_MODEL_ID));
|
||||
|
||||
let idx = policy.select_worker(
|
||||
&available,
|
||||
&SelectWorkerInfo {
|
||||
request_text: text,
|
||||
tokens: None, // HTTP doesn't have tokens, use gRPC for PrefixHash
|
||||
headers,
|
||||
hash_ring,
|
||||
},
|
||||
)?;
|
||||
let idx = policy
|
||||
.select_worker(
|
||||
&available,
|
||||
&SelectWorkerInfo {
|
||||
request_text: text,
|
||||
tokens: None, // HTTP doesn't have tokens, use gRPC for PrefixHash
|
||||
headers,
|
||||
hash_ring,
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
|
||||
// Record worker selection metric (Layer 3)
|
||||
Metrics::record_worker_selection(
|
||||
@@ -276,7 +278,10 @@ impl Router {
|
||||
is_stream: bool,
|
||||
text: &str,
|
||||
) -> Response {
|
||||
let worker = match self.select_worker_for_model(model_id, Some(text), headers) {
|
||||
let worker = match self
|
||||
.select_worker_for_model(model_id, Some(text), headers)
|
||||
.await
|
||||
{
|
||||
Some(w) => w,
|
||||
None => {
|
||||
return error::service_unavailable(
|
||||
|
||||
Reference in New Issue
Block a user