From 617e9b3bf8c047bf44568f65493249546b1a5deb Mon Sep 17 00:00:00 2001 From: fzyzcjy <5236035+fzyzcjy@users.noreply.github.com> Date: Thu, 11 Dec 2025 00:30:03 +0800 Subject: [PATCH] [model-gateway] support customizing Prometheus duration buckets (#14716) --- .../python/sglang_router/router_args.py | 7 +++++++ sgl-model-gateway/bindings/python/src/lib.rs | 5 +++++ sgl-model-gateway/src/main.rs | 8 +++++++ .../src/observability/metrics.rs | 21 +++++++++++++++---- 4 files changed, 37 insertions(+), 4 deletions(-) diff --git a/sgl-model-gateway/bindings/python/sglang_router/router_args.py b/sgl-model-gateway/bindings/python/sglang_router/router_args.py index 01d391d83..17b32c5f0 100644 --- a/sgl-model-gateway/bindings/python/sglang_router/router_args.py +++ b/sgl-model-gateway/bindings/python/sglang_router/router_args.py @@ -54,6 +54,7 @@ class RouterArgs: # Prometheus configuration prometheus_port: Optional[int] = None prometheus_host: Optional[str] = None + prometheus_duration_buckets: Optional[List[float]] = None # Request ID headers configuration request_id_headers: Optional[List[str]] = None # Request timeout in seconds @@ -342,6 +343,12 @@ class RouterArgs: default="0.0.0.0", help="Host address to bind the Prometheus metrics server. Supports IPv4, IPv6 (e.g., ::, ::1), or 0.0.0.0 for all interfaces", ) + parser.add_argument( + f"--{prefix}prometheus-duration-buckets", + type=float, + nargs="+", + help="Buckets for Prometheus duration metrics", + ) parser.add_argument( f"--{prefix}request-id-headers", type=str, diff --git a/sgl-model-gateway/bindings/python/src/lib.rs b/sgl-model-gateway/bindings/python/src/lib.rs index dbe29ebf4..f33caf581 100644 --- a/sgl-model-gateway/bindings/python/src/lib.rs +++ b/sgl-model-gateway/bindings/python/src/lib.rs @@ -177,6 +177,7 @@ struct Router { bootstrap_port_annotation: String, prometheus_port: Option, prometheus_host: Option, + prometheus_duration_buckets: Option>, request_timeout_secs: u64, request_id_headers: Option>, pd_disaggregation: bool, @@ -439,6 +440,7 @@ impl Router { bootstrap_port_annotation = String::from("sglang.ai/bootstrap-port"), prometheus_port = None, prometheus_host = None, + prometheus_duration_buckets = None, request_timeout_secs = 1800, request_id_headers = None, pd_disaggregation = false, @@ -516,6 +518,7 @@ impl Router { bootstrap_port_annotation: String, prometheus_port: Option, prometheus_host: Option, + prometheus_duration_buckets: Option>, request_timeout_secs: u64, request_id_headers: Option>, pd_disaggregation: bool, @@ -606,6 +609,7 @@ impl Router { bootstrap_port_annotation, prometheus_port, prometheus_host, + prometheus_duration_buckets, request_timeout_secs, request_id_headers, pd_disaggregation, @@ -695,6 +699,7 @@ impl Router { .prometheus_host .clone() .unwrap_or_else(|| "127.0.0.1".to_string()), + duration_buckets: self.prometheus_duration_buckets.clone(), }); let runtime = tokio::runtime::Runtime::new() diff --git a/sgl-model-gateway/src/main.rs b/sgl-model-gateway/src/main.rs index a66899070..e0c8f6bd9 100644 --- a/sgl-model-gateway/src/main.rs +++ b/sgl-model-gateway/src/main.rs @@ -214,6 +214,9 @@ struct CliArgs { #[arg(long, default_value = "0.0.0.0")] prometheus_host: String, + #[arg(long, num_args = 0..)] + prometheus_duration_buckets: Vec, + #[arg(long, num_args = 0..)] request_id_headers: Vec, @@ -681,6 +684,11 @@ impl CliArgs { let prometheus_config = Some(PrometheusConfig { port: self.prometheus_port, host: self.prometheus_host.clone(), + duration_buckets: if self.prometheus_duration_buckets.is_empty() { + None + } else { + Some(self.prometheus_duration_buckets.clone()) + }, }); ServerConfig { diff --git a/sgl-model-gateway/src/observability/metrics.rs b/sgl-model-gateway/src/observability/metrics.rs index c8d8ba11b..33fb6490c 100644 --- a/sgl-model-gateway/src/observability/metrics.rs +++ b/sgl-model-gateway/src/observability/metrics.rs @@ -10,6 +10,7 @@ use metrics_exporter_prometheus::{Matcher, PrometheusBuilder}; pub struct PrometheusConfig { pub port: u16, pub host: String, + pub duration_buckets: Option>, } impl Default for PrometheusConfig { @@ -17,6 +18,7 @@ impl Default for PrometheusConfig { Self { port: 29000, host: "0.0.0.0".to_string(), + duration_buckets: None, } } } @@ -274,10 +276,12 @@ pub fn start_prometheus(config: PrometheusConfig) { init_metrics(); let duration_matcher = Matcher::Suffix(String::from("duration_seconds")); - let duration_bucket = [ - 0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 15.0, 30.0, 45.0, - 60.0, 90.0, 120.0, 180.0, 240.0, - ]; + let duration_bucket: Vec = config.duration_buckets.unwrap_or_else(|| { + vec![ + 0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 15.0, 30.0, 45.0, + 60.0, 90.0, 120.0, 180.0, 240.0, + ] + }); let ip_addr: IpAddr = config .host @@ -713,6 +717,7 @@ mod tests { let config = PrometheusConfig { port: 8080, host: "127.0.0.1".to_string(), + duration_buckets: None, }; assert_eq!(config.port, 8080); assert_eq!(config.host, "127.0.0.1"); @@ -723,6 +728,7 @@ mod tests { let config = PrometheusConfig { port: 9090, host: "192.168.1.1".to_string(), + duration_buckets: None, }; let cloned = config.clone(); assert_eq!(cloned.port, config.port); @@ -737,6 +743,7 @@ mod tests { let config = PrometheusConfig { port: 29000, host: ip_str.to_string(), + duration_buckets: None, }; let ip_addr: IpAddr = config.host.parse().unwrap(); @@ -752,6 +759,7 @@ mod tests { let config = PrometheusConfig { port: 29000, host: ip_str.to_string(), + duration_buckets: None, }; let ip_addr: IpAddr = config.host.parse().unwrap(); @@ -767,6 +775,7 @@ mod tests { let config = PrometheusConfig { port: 29000, host: ip_str.to_string(), + duration_buckets: None, }; let ip_addr: IpAddr = config @@ -786,6 +795,7 @@ mod tests { let config = PrometheusConfig { port, host: host.to_string(), + duration_buckets: None, }; let ip_addr: IpAddr = config.host.parse().unwrap(); @@ -804,6 +814,7 @@ mod tests { let config = PrometheusConfig { port, host: "127.0.0.1".to_string(), + duration_buckets: None, }; let ip_addr: IpAddr = config.host.parse().unwrap(); @@ -969,6 +980,7 @@ mod tests { let config = PrometheusConfig { port, host: "127.0.0.1".to_string(), + duration_buckets: None, }; assert_eq!(config.port, port); @@ -980,6 +992,7 @@ mod tests { let config = PrometheusConfig { port: 29000, host: "127.0.0.1".to_string(), + duration_buckets: None, }; let ip_addr: IpAddr = config.host.parse().unwrap();