From 99d3bcdfed467daab54f4d36c96f9eb8a75af8b7 Mon Sep 17 00:00:00 2001 From: fzyzcjy <5236035+fzyzcjy@users.noreply.github.com> Date: Wed, 24 Dec 2025 10:50:38 +0800 Subject: [PATCH] [model-gateway] add back router worker health metric and fix init state (#15622) --- sgl-model-gateway/src/core/worker.rs | 1 + sgl-model-gateway/src/core/worker_builder.rs | 7 +++++-- sgl-model-gateway/src/observability/metrics.rs | 13 +++++++++++++ 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/sgl-model-gateway/src/core/worker.rs b/sgl-model-gateway/src/core/worker.rs index 99505c508..60e64e372 100644 --- a/sgl-model-gateway/src/core/worker.rs +++ b/sgl-model-gateway/src/core/worker.rs @@ -588,6 +588,7 @@ impl Worker for BasicWorker { fn set_healthy(&self, healthy: bool) { self.healthy.store(healthy, Ordering::Release); + Metrics::set_worker_health(self.url(), healthy); } async fn check_health_async(&self) -> WorkerResult<()> { diff --git a/sgl-model-gateway/src/core/worker_builder.rs b/sgl-model-gateway/src/core/worker_builder.rs index b0c714db7..cb6260829 100644 --- a/sgl-model-gateway/src/core/worker_builder.rs +++ b/sgl-model-gateway/src/core/worker_builder.rs @@ -9,7 +9,7 @@ use super::{ WorkerType, }, }; -use crate::routers::grpc::client::GrpcClient; +use crate::{observability::metrics::Metrics, routers::grpc::client::GrpcClient}; /// Builder for creating BasicWorker instances with fluent API pub struct BasicWorkerBuilder { @@ -187,11 +187,14 @@ impl BasicWorkerBuilder { None => OnceCell::new(), }); + let healthy = true; + Metrics::set_worker_health(&self.url, healthy); + BasicWorker { metadata, load_counter: Arc::new(AtomicUsize::new(0)), processed_counter: Arc::new(AtomicUsize::new(0)), - healthy: Arc::new(AtomicBool::new(true)), + healthy: Arc::new(AtomicBool::new(healthy)), consecutive_failures: Arc::new(AtomicUsize::new(0)), consecutive_successes: Arc::new(AtomicUsize::new(0)), circuit_breaker: CircuitBreaker::with_config_and_label( diff --git a/sgl-model-gateway/src/observability/metrics.rs b/sgl-model-gateway/src/observability/metrics.rs index 96416cef3..6865b939d 100644 --- a/sgl-model-gateway/src/observability/metrics.rs +++ b/sgl-model-gateway/src/observability/metrics.rs @@ -174,6 +174,10 @@ pub fn init_metrics() { "smg_worker_requests_active", "Currently running requests per worker" ); + describe_gauge!( + "smg_worker_health", + "Worker health status (1=healthy, 0=unhealthy)" + ); describe_counter!( "smg_worker_health_checks_total", "Health check results by worker_type and result" @@ -807,6 +811,15 @@ impl Metrics { .set(count as f64); } + /// Set worker health status + pub fn set_worker_health(worker_url: &str, healthy: bool) { + gauge!( + "smg_worker_health", + "worker" => worker_url.to_string() + ) + .set(if healthy { 1.0 } else { 0.0 }); + } + // ======================================================================== // Layer 3: Worker resilience metrics (circuit breaker) // ========================================================================