diff --git a/python/sglang/srt/managers/scheduler.py b/python/sglang/srt/managers/scheduler.py index 0c3fb83f5..45fbf5d65 100644 --- a/python/sglang/srt/managers/scheduler.py +++ b/python/sglang/srt/managers/scheduler.py @@ -2012,6 +2012,8 @@ class Scheduler( self.server_args, self.decode_mem_cache_buf_multiplier ) self.num_retracted_reqs = len(retracted_reqs) + if self.enable_metrics and (x := len(retracted_reqs)) > 0: + self.metrics_collector.increment_num_retracted_reqs(x) self.new_token_ratio = new_token_ratio for req in reqs_to_abort: abort_reason: FINISH_ABORT = req.to_finish diff --git a/python/sglang/srt/managers/scheduler_output_processor_mixin.py b/python/sglang/srt/managers/scheduler_output_processor_mixin.py index f96ed1dcd..c9fec5393 100644 --- a/python/sglang/srt/managers/scheduler_output_processor_mixin.py +++ b/python/sglang/srt/managers/scheduler_output_processor_mixin.py @@ -337,6 +337,8 @@ class SchedulerOutputProcessorMixin: self.num_generated_tokens += len(batch.reqs) if not batch.spec_algorithm.is_none(): self.update_spec_metrics(batch.batch_size(), result.num_accepted_tokens) + if self.enable_metrics: + self.metrics_collector.increment_cuda_graph_pass(value=can_run_cuda_graph) self.token_to_kv_pool_allocator.free_group_begin() diff --git a/python/sglang/srt/metrics/collector.py b/python/sglang/srt/metrics/collector.py index 7aa7bd511..8d2196ebd 100644 --- a/python/sglang/srt/metrics/collector.py +++ b/python/sglang/srt/metrics/collector.py @@ -325,11 +325,18 @@ class SchedulerMetricsCollector: ) # Retract + # TODO maybe remove this old gauge in favor of the new counter self.num_retracted_reqs = Gauge( name="sglang:num_retracted_reqs", documentation="The number of retracted requests.", labelnames=labels.keys(), ) + self.num_retracted_reqs_total = Counter( + # The name is `requests` instead of `reqs` to avoid dup name error + name="sglang:num_retracted_requests_total", + documentation="Total number of retracted requests.", + labelnames=labels.keys(), + ) self.num_paused_reqs = Gauge( name="sglang:num_paused_reqs", documentation="The number of paused requests by async weight sync.", @@ -605,12 +612,18 @@ class SchedulerMetricsCollector: labelnames=list(labels.keys()) + ["stage"], ) + # TODO maybe remove this old gauge in favor of the new counter self.is_cuda_graph = Gauge( name="sglang:is_cuda_graph", documentation="Whether the batch is using CUDA graph.", labelnames=labels.keys(), multiprocess_mode="mostrecent", ) + self.cuda_graph_passes_total = Counter( + name="sglang:cuda_graph_passes_total", + documentation="Total number of forward passes categorized by CUDA graph.", + labelnames=list(labels.keys()) + ["mode"], + ) self.new_token_ratio = Gauge( name="sglang:new_token_ratio", @@ -639,6 +652,14 @@ class SchedulerMetricsCollector: def observe_queue_time(self, latency: float) -> None: self._log_histogram(self.queue_time, latency) + def increment_num_retracted_reqs(self, num: int) -> None: + self.num_retracted_reqs_total.labels(**self.labels).inc(num) + + def increment_cuda_graph_pass(self, value: bool) -> None: + # leave room for piecewise cuda graph, etc + mode = "decode_cuda_graph" if value else "decode_none" + self.cuda_graph_passes_total.labels(**self.labels, mode=mode).inc(1) + def log_stats(self, stats: SchedulerStats) -> None: self._log_gauge(self.num_running_reqs, stats.num_running_reqs) self._log_gauge(self.num_used_tokens, stats.num_used_tokens)