diff --git a/python/sglang/srt/managers/scheduler_metrics_mixin.py b/python/sglang/srt/managers/scheduler_metrics_mixin.py index bf67ff5d2..59075c6ae 100644 --- a/python/sglang/srt/managers/scheduler_metrics_mixin.py +++ b/python/sglang/srt/managers/scheduler_metrics_mixin.py @@ -216,6 +216,11 @@ class SchedulerMetricsMixin: self.disagg_decode_transfer_queue.queue ) + self.metrics_collector.increment_realtime_tokens( + prefill_compute_tokens=adder.log_input_tokens, + prefill_cache_tokens=adder.log_hit_tokens, + ) + # Others self.calculate_utilization() self.metrics_collector.log_stats(self.stats) @@ -227,6 +232,7 @@ class SchedulerMetricsMixin: ): batch = running_batch or self.running_batch + last_num_generated_tokens = self.num_generated_tokens gap_latency = time.perf_counter() - self.last_decode_stats_tic self.last_decode_stats_tic = time.perf_counter() self.last_gen_throughput = self.num_generated_tokens / gap_latency @@ -367,6 +373,10 @@ class SchedulerMetricsMixin: self.disagg_decode_transfer_queue.queue ) + self.metrics_collector.increment_realtime_tokens( + decode_tokens=last_num_generated_tokens + ) + # Others self.calculate_utilization() self.metrics_collector.log_stats(self.stats) diff --git a/python/sglang/srt/metrics/collector.py b/python/sglang/srt/metrics/collector.py index 8d2196ebd..1c0417f14 100644 --- a/python/sglang/srt/metrics/collector.py +++ b/python/sglang/srt/metrics/collector.py @@ -632,6 +632,22 @@ class SchedulerMetricsCollector: multiprocess_mode="mostrecent", ) + self.realtime_prefill_compute_tokens_total = Counter( + name="sglang:realtime_prefill_compute_tokens_total", + documentation="Total number of prefill compute tokens processed (updated on each log interval).", + labelnames=labels.keys(), + ) + self.realtime_prefill_cache_tokens_total = Counter( + name="sglang:realtime_prefill_cache_tokens_total", + documentation="Total number of prefill cache tokens processed (updated on each log interval).", + labelnames=labels.keys(), + ) + self.realtime_decode_tokens_total = Counter( + name="sglang:realtime_decode_tokens_total", + documentation="Total number of decode tokens processed (updated on each log interval).", + labelnames=labels.keys(), + ) + def _log_gauge(self, gauge, data: Union[int, float]) -> None: # Convenience function for logging to gauge. gauge.labels(**self.labels).set(data) @@ -660,6 +676,17 @@ class SchedulerMetricsCollector: mode = "decode_cuda_graph" if value else "decode_none" self.cuda_graph_passes_total.labels(**self.labels, mode=mode).inc(1) + def increment_realtime_tokens( + self, prefill_compute_tokens=0, prefill_cache_tokens=0, decode_tokens=0 + ): + self.realtime_prefill_compute_tokens_total.labels(**self.labels).inc( + prefill_compute_tokens + ) + self.realtime_prefill_cache_tokens_total.labels(**self.labels).inc( + prefill_cache_tokens + ) + self.realtime_decode_tokens_total.labels(**self.labels).inc(decode_tokens) + def log_stats(self, stats: SchedulerStats) -> None: self._log_gauge(self.num_running_reqs, stats.num_running_reqs) self._log_gauge(self.num_used_tokens, stats.num_used_tokens)