[model-gateway] add back router worker health metric and fix init state (#15622)
This commit is contained in:
@@ -588,6 +588,7 @@ impl Worker for BasicWorker {
|
||||
|
||||
fn set_healthy(&self, healthy: bool) {
|
||||
self.healthy.store(healthy, Ordering::Release);
|
||||
Metrics::set_worker_health(self.url(), healthy);
|
||||
}
|
||||
|
||||
async fn check_health_async(&self) -> WorkerResult<()> {
|
||||
|
||||
@@ -9,7 +9,7 @@ use super::{
|
||||
WorkerType,
|
||||
},
|
||||
};
|
||||
use crate::routers::grpc::client::GrpcClient;
|
||||
use crate::{observability::metrics::Metrics, routers::grpc::client::GrpcClient};
|
||||
|
||||
/// Builder for creating BasicWorker instances with fluent API
|
||||
pub struct BasicWorkerBuilder {
|
||||
@@ -187,11 +187,14 @@ impl BasicWorkerBuilder {
|
||||
None => OnceCell::new(),
|
||||
});
|
||||
|
||||
let healthy = true;
|
||||
Metrics::set_worker_health(&self.url, healthy);
|
||||
|
||||
BasicWorker {
|
||||
metadata,
|
||||
load_counter: Arc::new(AtomicUsize::new(0)),
|
||||
processed_counter: Arc::new(AtomicUsize::new(0)),
|
||||
healthy: Arc::new(AtomicBool::new(true)),
|
||||
healthy: Arc::new(AtomicBool::new(healthy)),
|
||||
consecutive_failures: Arc::new(AtomicUsize::new(0)),
|
||||
consecutive_successes: Arc::new(AtomicUsize::new(0)),
|
||||
circuit_breaker: CircuitBreaker::with_config_and_label(
|
||||
|
||||
@@ -174,6 +174,10 @@ pub fn init_metrics() {
|
||||
"smg_worker_requests_active",
|
||||
"Currently running requests per worker"
|
||||
);
|
||||
describe_gauge!(
|
||||
"smg_worker_health",
|
||||
"Worker health status (1=healthy, 0=unhealthy)"
|
||||
);
|
||||
describe_counter!(
|
||||
"smg_worker_health_checks_total",
|
||||
"Health check results by worker_type and result"
|
||||
@@ -807,6 +811,15 @@ impl Metrics {
|
||||
.set(count as f64);
|
||||
}
|
||||
|
||||
/// Set worker health status
|
||||
pub fn set_worker_health(worker_url: &str, healthy: bool) {
|
||||
gauge!(
|
||||
"smg_worker_health",
|
||||
"worker" => worker_url.to_string()
|
||||
)
|
||||
.set(if healthy { 1.0 } else { 0.0 });
|
||||
}
|
||||
|
||||
// ========================================================================
|
||||
// Layer 3: Worker resilience metrics (circuit breaker)
|
||||
// ========================================================================
|
||||
|
||||
Reference in New Issue
Block a user