diff --git a/sgl-router/Cargo.toml b/sgl-router/Cargo.toml index c980f9913..25482ae27 100644 --- a/sgl-router/Cargo.toml +++ b/sgl-router/Cargo.toml @@ -84,6 +84,7 @@ subtle = "2.6" rustpython-parser = "0.4.0" num-traits = "0.2" openai-harmony = { git = "https://github.com/openai/harmony", tag = "v0.0.4" } +openmetrics-parser = "0.4.4" # gRPC and Protobuf dependencies tonic = { version = "0.14.2", features = ["gzip", "transport"] } diff --git a/sgl-router/src/core/metrics_aggregator.rs b/sgl-router/src/core/metrics_aggregator.rs new file mode 100644 index 000000000..360736095 --- /dev/null +++ b/sgl-router/src/core/metrics_aggregator.rs @@ -0,0 +1,91 @@ +use anyhow::ensure; +use openmetrics_parser::{MetricFamily, MetricsExposition, PrometheusType, PrometheusValue}; +use tracing::warn; + +#[derive(Debug)] +pub struct MetricPack { + pub labels: Vec<(String, String)>, + pub metrics_text: String, +} + +type PrometheusExposition = MetricsExposition; +type PrometheusFamily = MetricFamily; + +/// Aggregate Prometheus metrics scraped from multiple sources into a unified one +pub fn aggregate_metrics(metric_packs: Vec) -> anyhow::Result { + let mut expositions = vec![]; + for metric_pack in metric_packs { + let metrics_text = &metric_pack.metrics_text; + // Hacky workaround since the parser do not understand `:`, should improve later + let metrics_text = metrics_text.replace(":", "_"); + + let exposition = match openmetrics_parser::prometheus::parse_prometheus(&metrics_text) { + Ok(x) => x, + Err(err) => { + warn!( + "aggregate_metrics error when parsing text: pack={:?} err={:?}", + metric_pack, err + ); + continue; + } + }; + let exposition = transform_metrics(exposition, &metric_pack.labels); + expositions.push(exposition); + } + + let text = try_reduce(expositions.into_iter(), merge_exposition)? + .map(|x| format!("{x}")) + .unwrap_or_default(); + Ok(text) +} + +fn transform_metrics( + mut exposition: PrometheusExposition, + extra_labels: &[(String, String)], +) -> PrometheusExposition { + for family in exposition.families.values_mut() { + *family = family.with_labels(extra_labels.iter().map(|(k, v)| (k.as_str(), v.as_str()))); + } + exposition +} + +fn merge_exposition( + a: PrometheusExposition, + b: PrometheusExposition, +) -> anyhow::Result { + let mut ans = a; + for (name, family_b) in b.families.into_iter() { + let family_merged = if let Some(family_a) = ans.families.remove(&name) { + merge_family(family_a, family_b)? + } else { + family_b + }; + ans.families.insert(name, family_merged); + } + Ok(ans) +} + +fn merge_family(a: PrometheusFamily, b: PrometheusFamily) -> anyhow::Result { + ensure!( + a.get_label_names() == b.get_label_names(), + "Label names should agree a={:?} b={:?}", + a.get_label_names(), + b.get_label_names() + ); + a.with_samples(b.into_iter_samples()) + .map_err(|e| anyhow::anyhow!("failed to merge samples: {e:?}")) +} + +pub fn try_reduce(iterable: I, f: F) -> Result, E> +where + I: IntoIterator, + F: FnMut(T, T) -> Result, +{ + let mut it = iterable.into_iter(); + let first = match it.next() { + None => return Ok(None), + Some(x) => x, + }; + + Ok(Some(it.try_fold(first, f)?)) +} diff --git a/sgl-router/src/core/mod.rs b/sgl-router/src/core/mod.rs index 4d66ebbc7..87f1b3feb 100644 --- a/sgl-router/src/core/mod.rs +++ b/sgl-router/src/core/mod.rs @@ -11,6 +11,7 @@ pub mod circuit_breaker; pub mod error; pub mod job_queue; +pub mod metrics_aggregator; pub mod retry; pub mod token_bucket; pub mod worker; diff --git a/sgl-router/src/core/worker_manager.rs b/sgl-router/src/core/worker_manager.rs index f2cd76a75..5f94e55f1 100644 --- a/sgl-router/src/core/worker_manager.rs +++ b/sgl-router/src/core/worker_manager.rs @@ -5,7 +5,9 @@ use std::{collections::HashMap, sync::Arc, time::Duration}; +use axum::response::{IntoResponse, Response}; use futures::future; +use http::{Method, StatusCode}; use serde_json::Value; use tokio::{ sync::{watch, Mutex}, @@ -14,7 +16,7 @@ use tokio::{ use tracing::{debug, error, info, warn}; use crate::{ - core::{ConnectionMode, WorkerRegistry, WorkerType}, + core::{metrics_aggregator::MetricPack, ConnectionMode, WorkerRegistry, WorkerType}, policies::PolicyRegistry, protocols::worker_spec::{FlushCacheResult, WorkerLoadInfo, WorkerLoadsResult}, }; @@ -234,6 +236,90 @@ impl WorkerManager { failed, } } + + pub async fn get_engine_metrics( + worker_registry: &WorkerRegistry, + client: &reqwest::Client, + ) -> Response { + let engine_responses = + match Self::fan_out_simple_request(worker_registry, client, "metrics", Method::GET) + .await + { + Ok(x) => x, + Err(e) => return e, + }; + let engine_responses = engine_responses + .into_iter() + .map(|(worker_base_url, metrics_text)| MetricPack { + labels: vec![("worker_addr".into(), worker_base_url)], + metrics_text, + }) + .collect(); + let text = match crate::core::metrics_aggregator::aggregate_metrics(engine_responses) { + Ok(x) => x, + Err(e) => { + let error_msg = format!("Failed to aggregate metrics: {}", e); + return (StatusCode::INTERNAL_SERVER_ERROR, error_msg).into_response(); + } + }; + (StatusCode::OK, text).into_response() + } + + async fn fan_out_simple_request( + worker_registry: &WorkerRegistry, + client: &reqwest::Client, + endpoint: &str, + method: Method, + ) -> Result, Response> { + let workers = worker_registry.get_all(); + if workers.is_empty() { + return Err((StatusCode::SERVICE_UNAVAILABLE, "No available workers").into_response()); + } + + let mut responses = vec![]; + // May do parallel requests later + for worker in workers { + let worker_url = worker.url().to_string(); + + let url = format!("{}/{}", worker_url, endpoint); + let mut request_builder = match method { + Method::GET => client.get(url), + Method::POST => client.post(url), + _ => { + return Err(( + StatusCode::METHOD_NOT_ALLOWED, + "Unsupported method for simple routing", + ) + .into_response()) + } + }; + + if let Some(api_key) = worker.api_key() { + request_builder = + request_builder.header("Authorization", format!("Bearer {}", api_key)); + } + + match request_builder.send().await { + Ok(res) => { + let status = StatusCode::from_u16(res.status().as_u16()) + .unwrap_or(StatusCode::INTERNAL_SERVER_ERROR); + match res.text().await { + Ok(body_text) => { + if status.is_success() { + responses.push((worker_url, body_text)); + } + } + Err(e) => { + warn!("fan_out_simple_request failed when reading text: {}", e) + } + } + } + Err(e) => warn!("fan_out_simple_request failed when sending: {}", e), + } + } + + Ok(responses) + } } /// Load monitoring service that periodically fetches worker loads diff --git a/sgl-router/src/server.rs b/sgl-router/src/server.rs index c70ac9c46..df268863f 100644 --- a/sgl-router/src/server.rs +++ b/sgl-router/src/server.rs @@ -115,6 +115,10 @@ async fn health_generate(State(state): State>, req: Request) -> Re state.router.health_generate(req).await } +async fn engine_metrics(State(state): State>) -> Response { + WorkerManager::get_engine_metrics(&state.context.worker_registry, &state.context.client).await +} + async fn get_server_info(State(state): State>, req: Request) -> Response { state.router.get_server_info(req).await } @@ -641,6 +645,7 @@ pub fn build_app( .route("/readiness", get(readiness)) .route("/health", get(health)) .route("/health_generate", get(health_generate)) + .route("/engine_metrics", get(engine_metrics)) .route("/v1/models", get(v1_models)) .route("/get_model_info", get(get_model_info)) .route("/get_server_info", get(get_server_info)); diff --git a/sgl-router/tests/metrics_aggregator_test.rs b/sgl-router/tests/metrics_aggregator_test.rs new file mode 100644 index 000000000..fa3a78897 --- /dev/null +++ b/sgl-router/tests/metrics_aggregator_test.rs @@ -0,0 +1,271 @@ +use sglang_router_rs::core::metrics_aggregator::{aggregate_metrics, MetricPack}; + +#[test] +fn test_aggregate_simple() { + let pack1 = MetricPack { + labels: vec![("source".to_string(), "worker1".to_string())], + metrics_text: r#" +# HELP http_requests_total The total number of HTTP requests. +# TYPE http_requests_total counter +http_requests_total{method="post",code="200"} 1027 +http_requests_total{method="post",code="400"} 3 +"# + .to_string(), + }; + let pack2 = MetricPack { + labels: vec![("source".to_string(), "worker2".to_string())], + metrics_text: r#" +# HELP http_requests_total The total number of HTTP requests. +# TYPE http_requests_total counter +http_requests_total{method="post",code="200"} 500 +"# + .to_string(), + }; + + let result = aggregate_metrics(vec![pack1, pack2]).unwrap(); + let expected = r#"# HELP http_requests_total The total number of HTTP requests. +# TYPE http_requests_total counter +http_requests_total{code="200",method="post",source="worker1"} 1027 +http_requests_total{code="400",method="post",source="worker1"} 3 +http_requests_total{code="200",method="post",source="worker2"} 500 +"#; + assert_eq!(result.trim(), expected.trim()); +} + +#[test] +fn test_aggregate_multiple_metrics() { + let pack1 = MetricPack { + labels: vec![("source".to_string(), "w1".to_string())], + metrics_text: r#" +# TYPE metric_a gauge +metric_a{dim="x"} 1.0 +# TYPE metric_b_total counter +metric_b_total 10 +"# + .to_string(), + }; + let pack2 = MetricPack { + labels: vec![("source".to_string(), "w2".to_string())], + metrics_text: r#" +# TYPE metric_a gauge +metric_a{dim="y"} 2.0 +"# + .to_string(), + }; + + let result = aggregate_metrics(vec![pack1, pack2]).unwrap(); + let expected = r#"# TYPE metric_a gauge +metric_a{dim="x",source="w1"} 1 +metric_a{dim="y",source="w2"} 2 + +# TYPE metric_b_total counter +metric_b_total{source="w1"} 10 +"#; + assert_eq_sorted(&result, expected); +} + +#[test] +fn test_empty_input() { + let result = aggregate_metrics(vec![]).unwrap(); + assert_eq!(result, ""); +} + +#[test] +fn test_invalid_metrics_are_skipped() { + let pack1 = MetricPack { + labels: vec![("source".to_string(), "worker1".to_string())], + metrics_text: "invalid metrics text".to_string(), + }; + let pack2 = MetricPack { + labels: vec![("source".to_string(), "worker2".to_string())], + metrics_text: "# TYPE valid_metric gauge\nvalid_metric 123\n".to_string(), + }; + let result = aggregate_metrics(vec![pack1, pack2]).unwrap(); + let expected = r#"# TYPE valid_metric gauge +valid_metric{source="worker2"} 123 +"#; + assert_eq!(result.trim(), expected.trim()); +} + +#[test] +fn test_real() { + let pack1 = MetricPack { + labels: vec![("source".to_string(), "worker1".to_string())], + // https://docs.sglang.ai/references/production_metrics.html + metrics_text: r###"# HELP sglang:prompt_tokens_total Number of prefill tokens processed. +# TYPE sglang:prompt_tokens_total counter +sglang:prompt_tokens_total{model_name="meta-llama/Llama-3.1-8B-Instruct"} 8.128902e+06 +# HELP sglang:generation_tokens_total Number of generation tokens processed. +# TYPE sglang:generation_tokens_total counter +sglang:generation_tokens_total{model_name="meta-llama/Llama-3.1-8B-Instruct"} 7.557572e+06 +# HELP sglang:token_usage The token usage +# TYPE sglang:token_usage gauge +sglang:token_usage{model_name="meta-llama/Llama-3.1-8B-Instruct"} 0.28 +# HELP sglang:cache_hit_rate The cache hit rate +# TYPE sglang:cache_hit_rate gauge +sglang:cache_hit_rate{model_name="meta-llama/Llama-3.1-8B-Instruct"} 0.007507552643049313 +# HELP sglang:time_to_first_token_seconds Histogram of time to first token in seconds. +# TYPE sglang:time_to_first_token_seconds histogram +sglang:time_to_first_token_seconds_sum{model_name="meta-llama/Llama-3.1-8B-Instruct"} 2.3518979474117756e+06 +sglang:time_to_first_token_seconds_bucket{le="0.001",model_name="meta-llama/Llama-3.1-8B-Instruct"} 0.0 +sglang:time_to_first_token_seconds_bucket{le="0.005",model_name="meta-llama/Llama-3.1-8B-Instruct"} 0.0 +sglang:time_to_first_token_seconds_bucket{le="0.01",model_name="meta-llama/Llama-3.1-8B-Instruct"} 0.0 +sglang:time_to_first_token_seconds_bucket{le="+Inf",model_name="meta-llama/Llama-3.1-8B-Instruct"} 11008.0 +sglang:time_to_first_token_seconds_count{model_name="meta-llama/Llama-3.1-8B-Instruct"} 11008.0 +# HELP sglang:e2e_request_latency_seconds Histogram of End-to-end request latency in seconds +# TYPE sglang:e2e_request_latency_seconds histogram +sglang:e2e_request_latency_seconds_sum{model_name="meta-llama/Llama-3.1-8B-Instruct"} 3.116093850019932e+06 +sglang:e2e_request_latency_seconds_bucket{le="0.3",model_name="meta-llama/Llama-3.1-8B-Instruct"} 0.0 +sglang:e2e_request_latency_seconds_bucket{le="0.5",model_name="meta-llama/Llama-3.1-8B-Instruct"} 6.0 +sglang:e2e_request_latency_seconds_bucket{le="0.8",model_name="meta-llama/Llama-3.1-8B-Instruct"} 6.0 +sglang:e2e_request_latency_seconds_bucket{le="+Inf",model_name="meta-llama/Llama-3.1-8B-Instruct"} 11228.0 +sglang:e2e_request_latency_seconds_count{model_name="meta-llama/Llama-3.1-8B-Instruct"} 11228.0 +# HELP sglang:time_per_output_token_seconds Histogram of time per output token in seconds. +# TYPE sglang:time_per_output_token_seconds histogram +sglang:time_per_output_token_seconds_sum{model_name="meta-llama/Llama-3.1-8B-Instruct"} 866964.5791549598 +sglang:time_per_output_token_seconds_bucket{le="0.005",model_name="meta-llama/Llama-3.1-8B-Instruct"} 1.0 +sglang:time_per_output_token_seconds_bucket{le="0.01",model_name="meta-llama/Llama-3.1-8B-Instruct"} 73.0 +sglang:time_per_output_token_seconds_bucket{le="0.015",model_name="meta-llama/Llama-3.1-8B-Instruct"} 382.0 +sglang:time_per_output_token_seconds_bucket{le="+Inf",model_name="meta-llama/Llama-3.1-8B-Instruct"} 7.400757e+06 +sglang:time_per_output_token_seconds_count{model_name="meta-llama/Llama-3.1-8B-Instruct"} 7.400757e+06 +# HELP sglang:func_latency_seconds Function latency in seconds +# TYPE sglang:func_latency_seconds histogram +sglang:func_latency_seconds_sum{name="generate_request"} 4.514771912145079 +sglang:func_latency_seconds_bucket{le="0.05",name="generate_request"} 14006.0 +sglang:func_latency_seconds_bucket{le="0.07500000000000001",name="generate_request"} 14006.0 +sglang:func_latency_seconds_bucket{le="0.1125",name="generate_request"} 14006.0 +sglang:func_latency_seconds_bucket{le="0.16875",name="generate_request"} 14006.0 +sglang:func_latency_seconds_bucket{le="+Inf",name="generate_request"} 14007.0 +sglang:func_latency_seconds_count{name="generate_request"} 14007.0 +# HELP sglang:num_running_reqs The number of running requests +# TYPE sglang:num_running_reqs gauge +sglang:num_running_reqs{model_name="meta-llama/Llama-3.1-8B-Instruct"} 162.0 +# HELP sglang:num_used_tokens The number of used tokens +# TYPE sglang:num_used_tokens gauge +sglang:num_used_tokens{model_name="meta-llama/Llama-3.1-8B-Instruct"} 123859.0 +# HELP sglang:gen_throughput The generate throughput (token/s) +# TYPE sglang:gen_throughput gauge +sglang:gen_throughput{model_name="meta-llama/Llama-3.1-8B-Instruct"} 86.50814177726902 +# HELP sglang:num_queue_reqs The number of requests in the waiting queue +# TYPE sglang:num_queue_reqs gauge +sglang:num_queue_reqs{model_name="meta-llama/Llama-3.1-8B-Instruct"} 2826.0 +"###.to_string(), + }; + let pack2 = MetricPack { + labels: vec![("source".to_string(), "worker2".to_string())], + metrics_text: pack1.metrics_text.clone(), + }; + let result = aggregate_metrics(vec![pack1, pack2]).unwrap(); + let expected = r###"# HELP sglang_token_usage The token usage +# TYPE sglang_token_usage gauge +sglang_token_usage{model_name="meta-llama/Llama-3.1-8B-Instruct",source="worker1"} 0.28 +sglang_token_usage{model_name="meta-llama/Llama-3.1-8B-Instruct",source="worker2"} 0.28 + +# HELP sglang_time_to_first_token_seconds Histogram of time to first token in seconds. +# TYPE sglang_time_to_first_token_seconds histogram +sglang_time_to_first_token_seconds_bucket{model_name="meta-llama/Llama-3.1-8B-Instruct",source="worker1",le="0.001"} 0 +sglang_time_to_first_token_seconds_bucket{model_name="meta-llama/Llama-3.1-8B-Instruct",source="worker1",le="0.005"} 0 +sglang_time_to_first_token_seconds_bucket{model_name="meta-llama/Llama-3.1-8B-Instruct",source="worker1",le="0.01"} 0 +sglang_time_to_first_token_seconds_bucket{model_name="meta-llama/Llama-3.1-8B-Instruct",source="worker1",le="+Inf"} 11008 +sglang_time_to_first_token_seconds_sum{model_name="meta-llama/Llama-3.1-8B-Instruct",source="worker1"} 2351897.9474117756 +sglang_time_to_first_token_seconds_count{model_name="meta-llama/Llama-3.1-8B-Instruct",source="worker1"} 11008 +sglang_time_to_first_token_seconds_bucket{model_name="meta-llama/Llama-3.1-8B-Instruct",source="worker2",le="0.001"} 0 +sglang_time_to_first_token_seconds_bucket{model_name="meta-llama/Llama-3.1-8B-Instruct",source="worker2",le="0.005"} 0 +sglang_time_to_first_token_seconds_bucket{model_name="meta-llama/Llama-3.1-8B-Instruct",source="worker2",le="0.01"} 0 +sglang_time_to_first_token_seconds_bucket{model_name="meta-llama/Llama-3.1-8B-Instruct",source="worker2",le="+Inf"} 11008 +sglang_time_to_first_token_seconds_sum{model_name="meta-llama/Llama-3.1-8B-Instruct",source="worker2"} 2351897.9474117756 +sglang_time_to_first_token_seconds_count{model_name="meta-llama/Llama-3.1-8B-Instruct",source="worker2"} 11008 + +# HELP sglang_time_per_output_token_seconds Histogram of time per output token in seconds. +# TYPE sglang_time_per_output_token_seconds histogram +sglang_time_per_output_token_seconds_bucket{model_name="meta-llama/Llama-3.1-8B-Instruct",source="worker1",le="0.005"} 1 +sglang_time_per_output_token_seconds_bucket{model_name="meta-llama/Llama-3.1-8B-Instruct",source="worker1",le="0.01"} 73 +sglang_time_per_output_token_seconds_bucket{model_name="meta-llama/Llama-3.1-8B-Instruct",source="worker1",le="0.015"} 382 +sglang_time_per_output_token_seconds_bucket{model_name="meta-llama/Llama-3.1-8B-Instruct",source="worker1",le="+Inf"} 7400757 +sglang_time_per_output_token_seconds_sum{model_name="meta-llama/Llama-3.1-8B-Instruct",source="worker1"} 866964.5791549598 +sglang_time_per_output_token_seconds_count{model_name="meta-llama/Llama-3.1-8B-Instruct",source="worker1"} 7400757 +sglang_time_per_output_token_seconds_bucket{model_name="meta-llama/Llama-3.1-8B-Instruct",source="worker2",le="0.005"} 1 +sglang_time_per_output_token_seconds_bucket{model_name="meta-llama/Llama-3.1-8B-Instruct",source="worker2",le="0.01"} 73 +sglang_time_per_output_token_seconds_bucket{model_name="meta-llama/Llama-3.1-8B-Instruct",source="worker2",le="0.015"} 382 +sglang_time_per_output_token_seconds_bucket{model_name="meta-llama/Llama-3.1-8B-Instruct",source="worker2",le="+Inf"} 7400757 +sglang_time_per_output_token_seconds_sum{model_name="meta-llama/Llama-3.1-8B-Instruct",source="worker2"} 866964.5791549598 +sglang_time_per_output_token_seconds_count{model_name="meta-llama/Llama-3.1-8B-Instruct",source="worker2"} 7400757 + +# HELP sglang_func_latency_seconds Function latency in seconds +# TYPE sglang_func_latency_seconds histogram +sglang_func_latency_seconds_bucket{name="generate_request",source="worker1",le="0.05"} 14006 +sglang_func_latency_seconds_bucket{name="generate_request",source="worker1",le="0.07500000000000001"} 14006 +sglang_func_latency_seconds_bucket{name="generate_request",source="worker1",le="0.1125"} 14006 +sglang_func_latency_seconds_bucket{name="generate_request",source="worker1",le="0.16875"} 14006 +sglang_func_latency_seconds_bucket{name="generate_request",source="worker1",le="+Inf"} 14007 +sglang_func_latency_seconds_sum{name="generate_request",source="worker1"} 4.514771912145079 +sglang_func_latency_seconds_count{name="generate_request",source="worker1"} 14007 +sglang_func_latency_seconds_bucket{name="generate_request",source="worker2",le="0.05"} 14006 +sglang_func_latency_seconds_bucket{name="generate_request",source="worker2",le="0.07500000000000001"} 14006 +sglang_func_latency_seconds_bucket{name="generate_request",source="worker2",le="0.1125"} 14006 +sglang_func_latency_seconds_bucket{name="generate_request",source="worker2",le="0.16875"} 14006 +sglang_func_latency_seconds_bucket{name="generate_request",source="worker2",le="+Inf"} 14007 +sglang_func_latency_seconds_sum{name="generate_request",source="worker2"} 4.514771912145079 +sglang_func_latency_seconds_count{name="generate_request",source="worker2"} 14007 + +# HELP sglang_num_used_tokens The number of used tokens +# TYPE sglang_num_used_tokens gauge +sglang_num_used_tokens{model_name="meta-llama/Llama-3.1-8B-Instruct",source="worker1"} 123859 +sglang_num_used_tokens{model_name="meta-llama/Llama-3.1-8B-Instruct",source="worker2"} 123859 + +# HELP sglang_cache_hit_rate The cache hit rate +# TYPE sglang_cache_hit_rate gauge +sglang_cache_hit_rate{model_name="meta-llama/Llama-3.1-8B-Instruct",source="worker1"} 0.007507552643049313 +sglang_cache_hit_rate{model_name="meta-llama/Llama-3.1-8B-Instruct",source="worker2"} 0.007507552643049313 + +# HELP sglang_num_queue_reqs The number of requests in the waiting queue +# TYPE sglang_num_queue_reqs gauge +sglang_num_queue_reqs{model_name="meta-llama/Llama-3.1-8B-Instruct",source="worker1"} 2826 +sglang_num_queue_reqs{model_name="meta-llama/Llama-3.1-8B-Instruct",source="worker2"} 2826 + +# HELP sglang_generation_tokens_total Number of generation tokens processed. +# TYPE sglang_generation_tokens_total counter +sglang_generation_tokens_total{model_name="meta-llama/Llama-3.1-8B-Instruct",source="worker1"} 7557572 +sglang_generation_tokens_total{model_name="meta-llama/Llama-3.1-8B-Instruct",source="worker2"} 7557572 + +# HELP sglang_num_running_reqs The number of running requests +# TYPE sglang_num_running_reqs gauge +sglang_num_running_reqs{model_name="meta-llama/Llama-3.1-8B-Instruct",source="worker1"} 162 +sglang_num_running_reqs{model_name="meta-llama/Llama-3.1-8B-Instruct",source="worker2"} 162 + +# HELP sglang_e2e_request_latency_seconds Histogram of End-to-end request latency in seconds +# TYPE sglang_e2e_request_latency_seconds histogram +sglang_e2e_request_latency_seconds_bucket{model_name="meta-llama/Llama-3.1-8B-Instruct",source="worker1",le="0.3"} 0 +sglang_e2e_request_latency_seconds_bucket{model_name="meta-llama/Llama-3.1-8B-Instruct",source="worker1",le="0.5"} 6 +sglang_e2e_request_latency_seconds_bucket{model_name="meta-llama/Llama-3.1-8B-Instruct",source="worker1",le="0.8"} 6 +sglang_e2e_request_latency_seconds_bucket{model_name="meta-llama/Llama-3.1-8B-Instruct",source="worker1",le="+Inf"} 11228 +sglang_e2e_request_latency_seconds_sum{model_name="meta-llama/Llama-3.1-8B-Instruct",source="worker1"} 3116093.850019932 +sglang_e2e_request_latency_seconds_count{model_name="meta-llama/Llama-3.1-8B-Instruct",source="worker1"} 11228 +sglang_e2e_request_latency_seconds_bucket{model_name="meta-llama/Llama-3.1-8B-Instruct",source="worker2",le="0.3"} 0 +sglang_e2e_request_latency_seconds_bucket{model_name="meta-llama/Llama-3.1-8B-Instruct",source="worker2",le="0.5"} 6 +sglang_e2e_request_latency_seconds_bucket{model_name="meta-llama/Llama-3.1-8B-Instruct",source="worker2",le="0.8"} 6 +sglang_e2e_request_latency_seconds_bucket{model_name="meta-llama/Llama-3.1-8B-Instruct",source="worker2",le="+Inf"} 11228 +sglang_e2e_request_latency_seconds_sum{model_name="meta-llama/Llama-3.1-8B-Instruct",source="worker2"} 3116093.850019932 +sglang_e2e_request_latency_seconds_count{model_name="meta-llama/Llama-3.1-8B-Instruct",source="worker2"} 11228 + +# HELP sglang_gen_throughput The generate throughput (token/s) +# TYPE sglang_gen_throughput gauge +sglang_gen_throughput{model_name="meta-llama/Llama-3.1-8B-Instruct",source="worker1"} 86.50814177726902 +sglang_gen_throughput{model_name="meta-llama/Llama-3.1-8B-Instruct",source="worker2"} 86.50814177726902 + +# HELP sglang_prompt_tokens_total Number of prefill tokens processed. +# TYPE sglang_prompt_tokens_total counter +sglang_prompt_tokens_total{model_name="meta-llama/Llama-3.1-8B-Instruct",source="worker1"} 8128902 +sglang_prompt_tokens_total{model_name="meta-llama/Llama-3.1-8B-Instruct",source="worker2"} 8128902"###; + println!("result=\n{result}"); + assert_eq_sorted(result.trim(), expected.trim()); +} + +fn assert_eq_sorted(result: &str, expected: &str) { + // Split into lines and sort to handle BTreeMap ordering issues between test environments + let mut result_lines: Vec<_> = result.trim().lines().map(|l| l.trim()).collect(); + let mut expected_lines: Vec<_> = expected.trim().lines().map(|l| l.trim()).collect(); + result_lines.sort(); + expected_lines.sort(); + assert_eq!(result_lines, expected_lines); +}