From 0fa044ad63e38888ea64aed24b3f2b608ecd0112 Mon Sep 17 00:00:00 2001
From: Simo Lin <linsimo.mark@gmail.com>
Date: Sat, 13 Dec 2025 20:47:04 -0800
Subject: [PATCH] [model-gateway] Add circuit breaker and discovery watcher
 metrics (#15094)

---
 sgl-model-gateway/src/core/worker.rs          | 10 +++++
 .../src/observability/metrics.rs              | 41 +++++++++++++++++++
 sgl-model-gateway/src/service_discovery.rs    |  2 +
 3 files changed, 53 insertions(+)

diff --git a/sgl-model-gateway/src/core/worker.rs b/sgl-model-gateway/src/core/worker.rs
index 53dbe0186..9971bdba2 100644
--- a/sgl-model-gateway/src/core/worker.rs
+++ b/sgl-model-gateway/src/core/worker.rs
@@ -156,6 +156,16 @@ pub trait Worker: Send + Sync + fmt::Debug {
             CircuitState::HalfOpen => 2u8,
         };
         RouterMetrics::set_cb_state(self.url(), state_code);
+
+        // Update consecutive failures/successes gauges
+        RouterMetrics::set_cb_consecutive_failures(
+            self.url(),
+            self.circuit_breaker().failure_count(),
+        );
+        RouterMetrics::set_cb_consecutive_successes(
+            self.url(),
+            self.circuit_breaker().success_count(),
+        );
     }
 
     /// Check if this worker is DP-aware
diff --git a/sgl-model-gateway/src/observability/metrics.rs b/sgl-model-gateway/src/observability/metrics.rs
index 77e5bfa39..a67663d75 100644
--- a/sgl-model-gateway/src/observability/metrics.rs
+++ b/sgl-model-gateway/src/observability/metrics.rs
@@ -65,6 +65,23 @@ pub fn init_metrics() {
         "sgl_router_cb_outcomes_total",
         "Total number of circuit breaker outcomes by worker and outcome type (success/failure)"
     );
+    describe_gauge!(
+        "sgl_router_cb_consecutive_failures",
+        "Current consecutive failure count per worker circuit breaker"
+    );
+    describe_gauge!(
+        "sgl_router_cb_consecutive_successes",
+        "Current consecutive success count per worker circuit breaker"
+    );
+
+    describe_counter!(
+        "sgl_router_discovery_watcher_errors_total",
+        "Total number of Kubernetes watcher errors"
+    );
+    describe_counter!(
+        "sgl_router_discovery_watcher_restarts_total",
+        "Total number of Kubernetes watcher restarts"
+    );
 
     describe_gauge!(
         "sgl_router_active_workers",
@@ -461,9 +478,33 @@ impl RouterMetrics {
         .increment(1);
     }
 
+    pub fn set_cb_consecutive_failures(worker: &str, count: u32) {
+        gauge!("sgl_router_cb_consecutive_failures",
+            "worker" => worker.to_string()
+        )
+        .set(count as f64);
+    }
+
+    pub fn set_cb_consecutive_successes(worker: &str, count: u32) {
+        gauge!("sgl_router_cb_consecutive_successes",
+            "worker" => worker.to_string()
+        )
+        .set(count as f64);
+    }
+
+    pub fn record_discovery_watcher_error() {
+        counter!("sgl_router_discovery_watcher_errors_total").increment(1);
+    }
+
+    pub fn record_discovery_watcher_restart() {
+        counter!("sgl_router_discovery_watcher_restarts_total").increment(1);
+    }
+
     // TODO delete the metrics (instead of setting them to zero)
     pub fn remove_worker_metrics(worker_url: &str) {
         gauge!("sgl_router_cb_state","worker" => worker_url.to_string()).set(0.0);
+        gauge!("sgl_router_cb_consecutive_failures","worker" => worker_url.to_string()).set(0.0);
+        gauge!("sgl_router_cb_consecutive_successes","worker" => worker_url.to_string()).set(0.0);
         gauge!("sgl_router_worker_health","worker" => worker_url.to_string()).set(0.0);
         gauge!("sgl_router_running_requests","worker" => worker_url.to_string()).set(0.0);
         gauge!("sgl_router_tree_size","worker" => worker_url.to_string()).set(0.0);
diff --git a/sgl-model-gateway/src/service_discovery.rs b/sgl-model-gateway/src/service_discovery.rs
index 7ffb71799..4e659840b 100644
--- a/sgl-model-gateway/src/service_discovery.rs
+++ b/sgl-model-gateway/src/service_discovery.rs
@@ -300,6 +300,7 @@ pub async fn start_service_discovery(
                 }
                 Err(err) => {
                     error!("Error in Kubernetes watcher: {}", err);
+                    RouterMetrics::record_discovery_watcher_error();
                     warn!(
                         "Retrying in {} seconds with exponential backoff",
                         retry_delay.as_secs()
@@ -314,6 +315,7 @@ pub async fn start_service_discovery(
                 "Kubernetes watcher exited, restarting in {} seconds",
                 config_arc.check_interval.as_secs()
             );
+            RouterMetrics::record_discovery_watcher_restart();
             time::sleep(config_arc.check_interval).await;
         }
     });