chenchenghao/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

metrics: support customer labels specified in request header (#10143)

Browse Source
This commit is contained in:
Yingchun Lai
2025-09-14 20:00:08 -07:00
committed by GitHub
parent 8f6a175803
commit fc2c3a3d8e
11 changed files with 126 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/sglang/srt/metrics/collector.py
+23 -20
View File
@@ -12,7 +12,6 @@
# limitations under the License.
# ==============================================================================
"""Utilities for Prometheus Metrics Collection."""
import time
from dataclasses import dataclass, field
from enum import Enum
@@ -812,36 +811,38 @@ class TokenizerMetricsCollector:
buckets=bucket_time_to_first_token,
)
def _log_histogram(self, histogram, data: Union[int, float]) -> None:
histogram.labels(**self.labels).observe(data)
def observe_one_finished_request(
self,
labels: Dict[str, str],
prompt_tokens: int,
generation_tokens: int,
cached_tokens: int,
e2e_latency: float,
has_grammar: bool,
):
self.prompt_tokens_total.labels(**self.labels).inc(prompt_tokens)
self.generation_tokens_total.labels(**self.labels).inc(generation_tokens)
self.prompt_tokens_total.labels(**labels).inc(prompt_tokens)
self.generation_tokens_total.labels(**labels).inc(generation_tokens)
if cached_tokens > 0:
self.cached_tokens_total.labels(**self.labels).inc(cached_tokens)
self.num_requests_total.labels(**self.labels).inc(1)
self.cached_tokens_total.labels(**labels).inc(cached_tokens)
self.num_requests_total.labels(**labels).inc(1)
if has_grammar:
self.num_so_requests_total.labels(**self.labels).inc(1)
self._log_histogram(self.histogram_e2e_request_latency, e2e_latency)
self.num_so_requests_total.labels(**labels).inc(1)
self.histogram_e2e_request_latency.labels(**labels).observe(float(e2e_latency))
if self.collect_tokens_histogram:
self._log_histogram(self.prompt_tokens_histogram, prompt_tokens)
self._log_histogram(self.generation_tokens_histogram, generation_tokens)
self.prompt_tokens_histogram.labels(**labels).observe(float(prompt_tokens))
self.generation_tokens_histogram.labels(**labels).observe(
float(generation_tokens)
)
def observe_time_to_first_token(self, value: float, label: str = ""):
if label == "batch":
self.histogram_time_to_first_token_offline_batch.labels(
**self.labels
).observe(value)
def observe_time_to_first_token(
self, labels: Dict[str, str], value: float, type: str = ""
):
if type == "batch":
self.histogram_time_to_first_token_offline_batch.labels(**labels).observe(
value
)
else:
self.histogram_time_to_first_token.labels(**self.labels).observe(value)
self.histogram_time_to_first_token.labels(**labels).observe(value)
def check_time_to_first_token_straggler(self, value: float) -> bool:
his = self.histogram_time_to_first_token.labels(**self.labels)
@@ -856,12 +857,14 @@ class TokenizerMetricsCollector:
return value >= his._upper_bounds[i]
return False
def observe_inter_token_latency(self, internval: float, num_new_tokens: int):
def observe_inter_token_latency(
self, labels: Dict[str, str], internval: float, num_new_tokens: int
):
adjusted_interval = internval / num_new_tokens
# A faster version of the Histogram::observe which observes multiple values at the same time.
# reference: https://github.com/prometheus/client_python/blob/v0.21.1/prometheus_client/metrics.py#L639
his = self.histogram_inter_token_latency_seconds.labels(**self.labels)
his = self.histogram_inter_token_latency_seconds.labels(**labels)
his._sum.inc(internval)
for i, bound in enumerate(his._upper_bounds):