[mode;-gateway] add back fixes of incorrect metrics after worker removal (#15624)

sgl-model-gateway/src/core/worker_registry.rs

@@ -11,7 +11,10 @@ use std::sync::Arc;
 use dashmap::DashMap;
 use uuid::Uuid;
 
-use crate::core::{ConnectionMode, RuntimeType, Worker, WorkerType};
+use crate::{
+    core::{ConnectionMode, RuntimeType, Worker, WorkerType},
+    observability::metrics::Metrics,
+};
 
 /// Unique identifier for a worker
 #[derive(Debug, Clone, Hash, Eq, PartialEq)]
@@ -168,6 +171,7 @@ impl WorkerRegistry {
             }
 
             worker.set_healthy(false);
+            Metrics::remove_worker_metrics(worker.url());
 
             Some(worker)
         } else {

sgl-model-gateway/src/observability/metrics.rs

@@ -1032,6 +1032,21 @@ impl Metrics {
         )
         .increment(1);
     }
+
+    // ========================================================================
+    // Worker cleanup
+    // ========================================================================
+
+    pub fn remove_worker_metrics(worker_url: &str) {
+        gauge!("smg_worker_cb_consecutive_failures", "worker" => worker_url.to_string()).set(0.0);
+        gauge!("smg_worker_cb_consecutive_successes", "worker" => worker_url.to_string()).set(0.0);
+        gauge!("smg_worker_requests_active", "worker" => worker_url.to_string()).set(0.0);
+
+        // Zero for these metrics have special valid meaning, thus we set to -1 temporarily
+        // (and will remove them completely after https://github.com/metrics-rs/metrics/issues/653)
+        gauge!("smg_worker_cb_state", "worker" => worker_url.to_string()).set(-1.0);
+        gauge!("smg_worker_health","worker" => worker_url.to_string()).set(-1.0);
+    }
 }
 
 #[cfg(test)]