chenchenghao/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 11 Packages Projects Releases Wiki Activity

[model-gateway] support customizing Prometheus duration buckets (#14716)

Browse Source
This commit is contained in:
fzyzcjy
2025-12-11 00:30:03 +08:00
committed by GitHub
parent 83e35a7c29
commit 617e9b3bf8
4 changed files with 37 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
sgl-model-gateway/bindings/python/sglang_router/router_args.py
View File

@@ -54,6 +54,7 @@ class RouterArgs:
# Prometheus configuration
prometheus_port: Optional[int] = None
prometheus_host: Optional[str] = None
prometheus_duration_buckets: Optional[List[float]] = None
# Request ID headers configuration
request_id_headers: Optional[List[str]] = None
# Request timeout in seconds
@@ -342,6 +343,12 @@ class RouterArgs:
default="0.0.0.0",
help="Host address to bind the Prometheus metrics server. Supports IPv4, IPv6 (e.g., ::, ::1), or 0.0.0.0 for all interfaces",
)
parser.add_argument(
f"--{prefix}prometheus-duration-buckets",
type=float,
nargs="+",
help="Buckets for Prometheus duration metrics",
)
parser.add_argument(
f"--{prefix}request-id-headers",
type=str,

5
sgl-model-gateway/bindings/python/src/lib.rs
View File

@@ -177,6 +177,7 @@ struct Router {
bootstrap_port_annotation: String,
prometheus_port: Option<u16>,
prometheus_host: Option<String>,
prometheus_duration_buckets: Option<Vec<f64>>,
request_timeout_secs: u64,
request_id_headers: Option<Vec<String>>,
pd_disaggregation: bool,
@@ -439,6 +440,7 @@ impl Router {
bootstrap_port_annotation = String::from("sglang.ai/bootstrap-port"),
prometheus_port = None,
prometheus_host = None,
prometheus_duration_buckets = None,
request_timeout_secs = 1800,
request_id_headers = None,
pd_disaggregation = false,
@@ -516,6 +518,7 @@ impl Router {
bootstrap_port_annotation: String,
prometheus_port: Option<u16>,
prometheus_host: Option<String>,
prometheus_duration_buckets: Option<Vec<f64>>,
request_timeout_secs: u64,
request_id_headers: Option<Vec<String>>,
pd_disaggregation: bool,
@@ -606,6 +609,7 @@ impl Router {
bootstrap_port_annotation,
prometheus_port,
prometheus_host,
prometheus_duration_buckets,
request_timeout_secs,
request_id_headers,
pd_disaggregation,
@@ -695,6 +699,7 @@ impl Router {
.prometheus_host
.clone()
.unwrap_or_else(|| "127.0.0.1".to_string()),
duration_buckets: self.prometheus_duration_buckets.clone(),
});
let runtime = tokio::runtime::Runtime::new()

8
sgl-model-gateway/src/main.rs
View File

@@ -214,6 +214,9 @@ struct CliArgs {
#[arg(long, default_value = "0.0.0.0")]
prometheus_host: String,
#[arg(long, num_args = 0..)]
prometheus_duration_buckets: Vec<f64>,
#[arg(long, num_args = 0..)]
request_id_headers: Vec<String>,
@@ -681,6 +684,11 @@ impl CliArgs {
let prometheus_config = Some(PrometheusConfig {
port: self.prometheus_port,
host: self.prometheus_host.clone(),
duration_buckets: if self.prometheus_duration_buckets.is_empty() {
None
} else {
Some(self.prometheus_duration_buckets.clone())
},
});
ServerConfig {

21
sgl-model-gateway/src/observability/metrics.rs
View File

@@ -10,6 +10,7 @@ use metrics_exporter_prometheus::{Matcher, PrometheusBuilder};
pub struct PrometheusConfig {
pub port: u16,
pub host: String,
pub duration_buckets: Option<Vec<f64>>,
}
impl Default for PrometheusConfig {
@@ -17,6 +18,7 @@ impl Default for PrometheusConfig {
Self {
port: 29000,
host: "0.0.0.0".to_string(),
duration_buckets: None,
}
}
}
@@ -274,10 +276,12 @@ pub fn start_prometheus(config: PrometheusConfig) {
init_metrics();
let duration_matcher = Matcher::Suffix(String::from("duration_seconds"));
let duration_bucket = [
0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 15.0, 30.0, 45.0,
60.0, 90.0, 120.0, 180.0, 240.0,
];
let duration_bucket: Vec<f64> = config.duration_buckets.unwrap_or_else(|| {
vec![
0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 15.0, 30.0, 45.0,
60.0, 90.0, 120.0, 180.0, 240.0,
]
});
let ip_addr: IpAddr = config
.host
@@ -713,6 +717,7 @@ mod tests {
let config = PrometheusConfig {
port: 8080,
host: "127.0.0.1".to_string(),
duration_buckets: None,
};
assert_eq!(config.port, 8080);
assert_eq!(config.host, "127.0.0.1");
@@ -723,6 +728,7 @@ mod tests {
let config = PrometheusConfig {
port: 9090,
host: "192.168.1.1".to_string(),
duration_buckets: None,
};
let cloned = config.clone();
assert_eq!(cloned.port, config.port);
@@ -737,6 +743,7 @@ mod tests {
let config = PrometheusConfig {
port: 29000,
host: ip_str.to_string(),
duration_buckets: None,
};
let ip_addr: IpAddr = config.host.parse().unwrap();
@@ -752,6 +759,7 @@ mod tests {
let config = PrometheusConfig {
port: 29000,
host: ip_str.to_string(),
duration_buckets: None,
};
let ip_addr: IpAddr = config.host.parse().unwrap();
@@ -767,6 +775,7 @@ mod tests {
let config = PrometheusConfig {
port: 29000,
host: ip_str.to_string(),
duration_buckets: None,
};
let ip_addr: IpAddr = config
@@ -786,6 +795,7 @@ mod tests {
let config = PrometheusConfig {
port,
host: host.to_string(),
duration_buckets: None,
};
let ip_addr: IpAddr = config.host.parse().unwrap();
@@ -804,6 +814,7 @@ mod tests {
let config = PrometheusConfig {
port,
host: "127.0.0.1".to_string(),
duration_buckets: None,
};
let ip_addr: IpAddr = config.host.parse().unwrap();
@@ -969,6 +980,7 @@ mod tests {
let config = PrometheusConfig {
port,
host: "127.0.0.1".to_string(),
duration_buckets: None,
};
assert_eq!(config.port, port);
@@ -980,6 +992,7 @@ mod tests {
let config = PrometheusConfig {
port: 29000,
host: "127.0.0.1".to_string(),
duration_buckets: None,
};
let ip_addr: IpAddr = config.host.parse().unwrap();