[model-gateway] support engine response http status statistics in router (#14712)

sgl-model-gateway/src/observability/metrics.rs

@@ -36,6 +36,10 @@ pub fn init_metrics() {
         "sgl_router_request_errors_total",
         "Total number of request errors by route and error type"
     );
+    describe_counter!(
+        "sgl_router_upstream_http_responses_total",
+        "Total number of upstream engine HTTP responses by status code"
+    );
     describe_counter!(
         "sgl_router_retries_total",
         "Total number of request retries by route"
@@ -325,6 +329,15 @@ impl RouterMetrics {
         .increment(1);
     }
 
+    // TODO unify metric names
+    pub fn record_upstream_http_response(route: &str, status_code: u16) {
+        counter!("sgl_router_upstream_http_responses_total",
+            "route" => route.to_string(),
+            "status_code" => status_code.to_string()
+        )
+        .increment(1);
+    }
+
     pub fn record_retry(route: &str) {
         counter!("sgl_router_retries_total",
             "route" => route.to_string()

sgl-model-gateway/src/routers/http/router.rs

@@ -531,6 +531,8 @@ impl Router {
             }
         };
 
+        RouterMetrics::record_upstream_http_response(route, res.status().as_u16());
+
         let status = StatusCode::from_u16(res.status().as_u16())
             .unwrap_or(StatusCode::INTERNAL_SERVER_ERROR);