[model-gateway] add --disable-health-check option to skip worker health probes (#17002)

This commit is contained in:
Ziwen Zhao
2026-01-13 17:28:32 -08:00
committed by GitHub
parent 5938c3b06a
commit cf25852a1d
17 changed files with 159 additions and 14 deletions
+4
View File
@@ -590,6 +590,9 @@ impl JobQueue {
health_failure_threshold: router_config
.health_check
.failure_threshold,
disable_health_check: router_config
.health_check
.disable_health_check,
max_connection_attempts: router_config
.health_check
.success_threshold
@@ -652,6 +655,7 @@ impl JobQueue {
health_check_interval_secs: router_config.health_check.check_interval_secs,
health_success_threshold: router_config.health_check.success_threshold,
health_failure_threshold: router_config.health_check.failure_threshold,
disable_health_check: router_config.health_check.disable_health_check,
max_connection_attempts: router_config.health_check.success_threshold * 10,
dp_aware: router_config.dp_aware,
};
@@ -60,6 +60,7 @@ impl StepExecutor<ExternalWorkerWorkflowData> for CreateExternalWorkersStep {
endpoint: cfg.endpoint.clone(),
failure_threshold: cfg.failure_threshold,
success_threshold: cfg.success_threshold,
disable_health_check: cfg.disable_health_check || config.disable_health_check,
}
};
@@ -98,7 +99,11 @@ impl StepExecutor<ExternalWorkerWorkflowData> for CreateExternalWorkersStep {
}
let worker = Arc::new(builder.build()) as Arc<dyn Worker>;
worker.set_healthy(false);
if health_config.disable_health_check {
worker.set_healthy(true);
} else {
worker.set_healthy(false);
}
info!(
"Created wildcard worker at {} (accepts any model, user auth forwarded)",
@@ -132,7 +137,11 @@ impl StepExecutor<ExternalWorkerWorkflowData> for CreateExternalWorkersStep {
}
let worker = Arc::new(builder.build()) as Arc<dyn Worker>;
worker.set_healthy(false);
if health_config.disable_health_check {
worker.set_healthy(true);
} else {
worker.set_healthy(false);
}
debug!(
"Created external worker for model {} at {}",
@@ -106,7 +106,7 @@ impl StepExecutor<LocalWorkerWorkflowData> for CreateLocalWorkerStep {
let circuit_breaker_config = build_circuit_breaker_config(app_context);
// Build health config
let health_config = build_health_config(app_context);
let health_config = build_health_config(app_context, config);
// Normalize URL
let normalized_url = normalize_url(&config.url, connection_mode);
@@ -270,7 +270,7 @@ fn build_circuit_breaker_config(app_context: &AppContext) -> CircuitBreakerConfi
}
}
fn build_health_config(app_context: &AppContext) -> HealthConfig {
fn build_health_config(app_context: &AppContext, config: &WorkerConfigRequest) -> HealthConfig {
let cfg = &app_context.router_config.health_check;
HealthConfig {
timeout_secs: cfg.timeout_secs,
@@ -278,6 +278,7 @@ fn build_health_config(app_context: &AppContext) -> HealthConfig {
endpoint: cfg.endpoint.clone(),
failure_threshold: cfg.failure_threshold,
success_threshold: cfg.success_threshold,
disable_health_check: cfg.disable_health_check || config.disable_health_check,
}
}
@@ -334,7 +335,11 @@ fn create_dp_aware_workers(
}
let worker = Arc::new(builder.build()) as Arc<dyn Worker>;
worker.set_healthy(false);
if health_config.disable_health_check {
worker.set_healthy(true);
} else {
worker.set_healthy(false);
}
workers.push(worker);
debug!(
@@ -358,6 +363,8 @@ fn create_single_worker(
config: &WorkerConfigRequest,
final_labels: &HashMap<String, String>,
) -> Vec<Arc<dyn Worker>> {
let health_check_disabled = health_config.disable_health_check;
let mut builder = BasicWorkerBuilder::new(normalized_url.to_string())
.model(model_card)
.worker_type(worker_type)
@@ -374,7 +381,11 @@ fn create_single_worker(
}
let worker = Arc::new(builder.build()) as Arc<dyn Worker>;
worker.set_healthy(false);
if health_check_disabled {
worker.set_healthy(true);
} else {
worker.set_healthy(false);
}
debug!(
"Created worker object for {} ({:?}) with {} labels",
@@ -80,6 +80,9 @@ impl StepExecutor<WorkerUpdateWorkflowData> for UpdateWorkerPropertiesStep {
success_threshold: request
.health_success_threshold
.unwrap_or(existing_health.success_threshold),
disable_health_check: request
.disable_health_check
.unwrap_or(existing_health.disable_health_check),
};
// Determine API key: use new one if provided, otherwise keep existing
+15
View File
@@ -531,6 +531,8 @@ pub struct HealthConfig {
pub failure_threshold: u32,
/// Number of consecutive successes before marking healthy
pub success_threshold: u32,
/// Whether to disable health checks for this worker
pub disable_health_check: bool,
}
impl Default for HealthConfig {
@@ -541,6 +543,7 @@ impl Default for HealthConfig {
endpoint: "/health".to_string(),
failure_threshold: 3,
success_threshold: 2,
disable_health_check: false,
}
}
}
@@ -698,6 +701,13 @@ impl Worker for BasicWorker {
}
async fn check_health_async(&self) -> WorkerResult<()> {
if self.metadata.health_config.disable_health_check {
if !self.is_healthy() {
self.set_healthy(true);
}
return Ok(());
}
let health_result = match &self.metadata.connection_mode {
ConnectionMode::Http => self.http_health_check().await?,
ConnectionMode::Grpc { .. } => self.grpc_health_check().await?,
@@ -1249,6 +1259,7 @@ pub fn worker_to_info(worker: &Arc<dyn Worker>) -> WorkerInfo {
chat_template: worker.chat_template(model_id).map(String::from),
bootstrap_port,
metadata: worker.metadata().labels.clone(),
disable_health_check: worker.metadata().health_config.disable_health_check,
job_status: None,
}
}
@@ -1322,6 +1333,7 @@ mod tests {
assert_eq!(config.endpoint, "/health");
assert_eq!(config.failure_threshold, 3);
assert_eq!(config.success_threshold, 2);
assert!(!config.disable_health_check);
}
#[test]
@@ -1332,12 +1344,14 @@ mod tests {
endpoint: "/healthz".to_string(),
failure_threshold: 5,
success_threshold: 3,
disable_health_check: true,
};
assert_eq!(config.timeout_secs, 10);
assert_eq!(config.check_interval_secs, 60);
assert_eq!(config.endpoint, "/healthz");
assert_eq!(config.failure_threshold, 5);
assert_eq!(config.success_threshold, 3);
assert!(config.disable_health_check);
}
#[test]
@@ -1376,6 +1390,7 @@ mod tests {
endpoint: "/custom-health".to_string(),
failure_threshold: 4,
success_threshold: 2,
disable_health_check: false,
};
use crate::core::BasicWorkerBuilder;
@@ -397,6 +397,7 @@ mod tests {
check_interval_secs: 60,
failure_threshold: 3,
success_threshold: 2,
disable_health_check: false,
};
let cb_config = CircuitBreakerConfig {
@@ -489,6 +490,7 @@ mod tests {
check_interval_secs: 45,
failure_threshold: 5,
success_threshold: 3,
disable_health_check: false,
};
let worker = DPAwareWorkerBuilder::new("http://localhost:8080", 3, 16)
@@ -671,6 +671,7 @@ impl WorkerRegistry {
// This is especially important when there are many workers
let health_futures: Vec<_> = workers
.iter()
.filter(|worker| !worker.metadata().health_config.disable_health_check)
.map(|worker| {
let worker = worker.clone();
async move {