From 4d64f150862fbb1ec3e21c22ba2681638937a533 Mon Sep 17 00:00:00 2001
From: fzyzcjy <5236035+fzyzcjy@users.noreply.github.com>
Date: Wed, 24 Dec 2025 10:50:03 +0800
Subject: [PATCH] [mode;-gateway] add back fixes of incorrect metrics after
 worker removal (#15624)

---
 sgl-model-gateway/src/core/worker_registry.rs  |  6 +++++-
 sgl-model-gateway/src/observability/metrics.rs | 15 +++++++++++++++
 2 files changed, 20 insertions(+), 1 deletion(-)

diff --git a/sgl-model-gateway/src/core/worker_registry.rs b/sgl-model-gateway/src/core/worker_registry.rs
index c74eca118..0db95b8b5 100644
--- a/sgl-model-gateway/src/core/worker_registry.rs
+++ b/sgl-model-gateway/src/core/worker_registry.rs
@@ -11,7 +11,10 @@ use std::sync::Arc;
 use dashmap::DashMap;
 use uuid::Uuid;
 
-use crate::core::{ConnectionMode, RuntimeType, Worker, WorkerType};
+use crate::{
+    core::{ConnectionMode, RuntimeType, Worker, WorkerType},
+    observability::metrics::Metrics,
+};
 
 /// Unique identifier for a worker
 #[derive(Debug, Clone, Hash, Eq, PartialEq)]
@@ -168,6 +171,7 @@ impl WorkerRegistry {
             }
 
             worker.set_healthy(false);
+            Metrics::remove_worker_metrics(worker.url());
 
             Some(worker)
         } else {
diff --git a/sgl-model-gateway/src/observability/metrics.rs b/sgl-model-gateway/src/observability/metrics.rs
index d68f6921f..96416cef3 100644
--- a/sgl-model-gateway/src/observability/metrics.rs
+++ b/sgl-model-gateway/src/observability/metrics.rs
@@ -1032,6 +1032,21 @@ impl Metrics {
         )
         .increment(1);
     }
+
+    // ========================================================================
+    // Worker cleanup
+    // ========================================================================
+
+    pub fn remove_worker_metrics(worker_url: &str) {
+        gauge!("smg_worker_cb_consecutive_failures", "worker" => worker_url.to_string()).set(0.0);
+        gauge!("smg_worker_cb_consecutive_successes", "worker" => worker_url.to_string()).set(0.0);
+        gauge!("smg_worker_requests_active", "worker" => worker_url.to_string()).set(0.0);
+
+        // Zero for these metrics have special valid meaning, thus we set to -1 temporarily
+        // (and will remove them completely after https://github.com/metrics-rs/metrics/issues/653)
+        gauge!("smg_worker_cb_state", "worker" => worker_url.to_string()).set(-1.0);
+        gauge!("smg_worker_health","worker" => worker_url.to_string()).set(-1.0);
+    }
 }
 
 #[cfg(test)]