[model-gateway] add back router worker health metric and fix init state (#15622)

This commit is contained in:
fzyzcjy
2025-12-24 10:50:38 +08:00
committed by GitHub
parent 4d64f15086
commit 99d3bcdfed
3 changed files with 19 additions and 2 deletions

View File

@@ -588,6 +588,7 @@ impl Worker for BasicWorker {
fn set_healthy(&self, healthy: bool) {
self.healthy.store(healthy, Ordering::Release);
Metrics::set_worker_health(self.url(), healthy);
}
async fn check_health_async(&self) -> WorkerResult<()> {

View File

@@ -9,7 +9,7 @@ use super::{
WorkerType,
},
};
use crate::routers::grpc::client::GrpcClient;
use crate::{observability::metrics::Metrics, routers::grpc::client::GrpcClient};
/// Builder for creating BasicWorker instances with fluent API
pub struct BasicWorkerBuilder {
@@ -187,11 +187,14 @@ impl BasicWorkerBuilder {
None => OnceCell::new(),
});
let healthy = true;
Metrics::set_worker_health(&self.url, healthy);
BasicWorker {
metadata,
load_counter: Arc::new(AtomicUsize::new(0)),
processed_counter: Arc::new(AtomicUsize::new(0)),
healthy: Arc::new(AtomicBool::new(true)),
healthy: Arc::new(AtomicBool::new(healthy)),
consecutive_failures: Arc::new(AtomicUsize::new(0)),
consecutive_successes: Arc::new(AtomicUsize::new(0)),
circuit_breaker: CircuitBreaker::with_config_and_label(

View File

@@ -174,6 +174,10 @@ pub fn init_metrics() {
"smg_worker_requests_active",
"Currently running requests per worker"
);
describe_gauge!(
"smg_worker_health",
"Worker health status (1=healthy, 0=unhealthy)"
);
describe_counter!(
"smg_worker_health_checks_total",
"Health check results by worker_type and result"
@@ -807,6 +811,15 @@ impl Metrics {
.set(count as f64);
}
/// Set worker health status
pub fn set_worker_health(worker_url: &str, healthy: bool) {
gauge!(
"smg_worker_health",
"worker" => worker_url.to_string()
)
.set(if healthy { 1.0 } else { 0.0 });
}
// ========================================================================
// Layer 3: Worker resilience metrics (circuit breaker)
// ========================================================================