chenchenghao/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 11 Packages Projects Releases Wiki Activity

[Auto Sync] Add max_total_num_tokens metric: Update scheduler_metrics_mixin.py, collector.py (20251202) (#14256)

Browse Source 
Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
Co-authored-by: Dan Zheng <dzheng@x.ai>
This commit is contained in:
Lianmin Zheng
2025-12-01 16:34:34 -08:00
committed by GitHub
parent 1da59e8304
commit 796d82b107
2 changed files with 15 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
python/sglang/srt/managers/scheduler_metrics_mixin.py
View File

@@ -172,6 +172,8 @@ class SchedulerMetricsMixin:
self.stats.num_grammar_queue_reqs = len(self.grammar_queue)
self.stats.cache_hit_rate = cache_hit_rate
self.stats.max_total_num_tokens = self.max_total_num_tokens
# Retract
self.stats.num_retracted_reqs = self.num_retracted_reqs
self.stats.num_paused_reqs = self.num_paused_reqs
@@ -320,6 +322,8 @@ class SchedulerMetricsMixin:
self.stats.num_grammar_queue_reqs = len(self.grammar_queue)
self.stats.cache_hit_rate = cache_hit_rate
self.stats.max_total_num_tokens = self.max_total_num_tokens
# Speculative decoding
self.stats.spec_accept_rate = spec_accept_rate
self.stats.spec_accept_length = spec_accept_length

11
python/sglang/srt/metrics/collector.py
View File

@@ -184,6 +184,8 @@ class SchedulerStats:
num_running_reqs_offline_batch: int = 0
cache_hit_rate: float = 0.0
max_total_num_tokens: int = 0
# Speculative decoding
spec_accept_length: float = 0.0
spec_accept_rate: float = 0.0
@@ -294,6 +296,13 @@ class SchedulerMetricsCollector:
multiprocess_mode="mostrecent",
)
self.max_total_num_tokens = Gauge(
name="sglang:max_total_num_tokens",
documentation="Maximum total number of tokens in the KV cache pool.",
labelnames=labels.keys(),
multiprocess_mode="mostrecent",
)
# Speculative decoding
self.spec_accept_length = Gauge(
name="sglang:spec_accept_length",
@@ -640,6 +649,8 @@ class SchedulerMetricsCollector:
)
self._log_gauge(self.cache_hit_rate, stats.cache_hit_rate)
self._log_gauge(self.max_total_num_tokens, stats.max_total_num_tokens)
# Speculative decoding
self._log_gauge(self.spec_accept_length, stats.spec_accept_length)
self._log_gauge(self.spec_accept_rate, stats.spec_accept_rate)