diff --git a/python/sglang/srt/managers/scheduler_metrics_mixin.py b/python/sglang/srt/managers/scheduler_metrics_mixin.py index bc990f70a..da2e220a1 100644 --- a/python/sglang/srt/managers/scheduler_metrics_mixin.py +++ b/python/sglang/srt/managers/scheduler_metrics_mixin.py @@ -172,6 +172,8 @@ class SchedulerMetricsMixin: self.stats.num_grammar_queue_reqs = len(self.grammar_queue) self.stats.cache_hit_rate = cache_hit_rate + self.stats.max_total_num_tokens = self.max_total_num_tokens + # Retract self.stats.num_retracted_reqs = self.num_retracted_reqs self.stats.num_paused_reqs = self.num_paused_reqs @@ -320,6 +322,8 @@ class SchedulerMetricsMixin: self.stats.num_grammar_queue_reqs = len(self.grammar_queue) self.stats.cache_hit_rate = cache_hit_rate + self.stats.max_total_num_tokens = self.max_total_num_tokens + # Speculative decoding self.stats.spec_accept_rate = spec_accept_rate self.stats.spec_accept_length = spec_accept_length diff --git a/python/sglang/srt/metrics/collector.py b/python/sglang/srt/metrics/collector.py index b4af2288f..ef0d73cc9 100644 --- a/python/sglang/srt/metrics/collector.py +++ b/python/sglang/srt/metrics/collector.py @@ -184,6 +184,8 @@ class SchedulerStats: num_running_reqs_offline_batch: int = 0 cache_hit_rate: float = 0.0 + max_total_num_tokens: int = 0 + # Speculative decoding spec_accept_length: float = 0.0 spec_accept_rate: float = 0.0 @@ -294,6 +296,13 @@ class SchedulerMetricsCollector: multiprocess_mode="mostrecent", ) + self.max_total_num_tokens = Gauge( + name="sglang:max_total_num_tokens", + documentation="Maximum total number of tokens in the KV cache pool.", + labelnames=labels.keys(), + multiprocess_mode="mostrecent", + ) + # Speculative decoding self.spec_accept_length = Gauge( name="sglang:spec_accept_length", @@ -640,6 +649,8 @@ class SchedulerMetricsCollector: ) self._log_gauge(self.cache_hit_rate, stats.cache_hit_rate) + self._log_gauge(self.max_total_num_tokens, stats.max_total_num_tokens) + # Speculative decoding self._log_gauge(self.spec_accept_length, stats.spec_accept_length) self._log_gauge(self.spec_accept_rate, stats.spec_accept_rate)