diff --git a/python/sglang/srt/managers/scheduler.py b/python/sglang/srt/managers/scheduler.py index 391c15867..a814ab1cd 100644 --- a/python/sglang/srt/managers/scheduler.py +++ b/python/sglang/srt/managers/scheduler.py @@ -2077,8 +2077,16 @@ class Scheduler( new_token_gained = new_available_tokens - old_available_tokens self.num_retracted_reqs = len(retracted_reqs) - if self.enable_metrics and (x := len(retracted_reqs)) > 0: - self.metrics_collector.increment_num_retracted_reqs(x) + if self.enable_metrics and len(retracted_reqs) > 0: + self.metrics_collector.increment_retracted_reqs( + num_retracted_reqs=len(retracted_reqs), + num_retracted_input_tokens=sum( + len(r.origin_input_ids) for r in retracted_reqs + ), + num_retracted_output_tokens=sum( + len(r.output_ids) for r in retracted_reqs + ), + ) self.new_token_ratio = new_token_ratio for req in reqs_to_abort: abort_reason: FINISH_ABORT = req.to_finish diff --git a/python/sglang/srt/metrics/collector.py b/python/sglang/srt/metrics/collector.py index 92736e894..edaada3b4 100644 --- a/python/sglang/srt/metrics/collector.py +++ b/python/sglang/srt/metrics/collector.py @@ -379,6 +379,16 @@ class SchedulerMetricsCollector: documentation="Total number of retracted requests.", labelnames=labels.keys(), ) + self.num_retracted_input_tokens_total = Counter( + name="sglang:num_retracted_input_tokens_total", + documentation="Total number of retracted input tokens.", + labelnames=labels.keys(), + ) + self.num_retracted_output_tokens_total = Counter( + name="sglang:num_retracted_output_tokens_total", + documentation="Total number of retracted output tokens.", + labelnames=labels.keys(), + ) self.num_paused_reqs = Gauge( name="sglang:num_paused_reqs", documentation="The number of paused requests by async weight sync.", @@ -731,8 +741,19 @@ class SchedulerMetricsCollector: def observe_queue_time(self, latency: float) -> None: self._log_histogram(self.queue_time, latency) - def increment_num_retracted_reqs(self, num: int) -> None: - self.num_retracted_reqs_total.labels(**self.labels).inc(num) + def increment_retracted_reqs( + self, + num_retracted_reqs: int, + num_retracted_input_tokens: int, + num_retracted_output_tokens: int, + ) -> None: + self.num_retracted_reqs_total.labels(**self.labels).inc(num_retracted_reqs) + self.num_retracted_input_tokens_total.labels(**self.labels).inc( + num_retracted_input_tokens + ) + self.num_retracted_output_tokens_total.labels(**self.labels).inc( + num_retracted_output_tokens + ) def increment_cuda_graph_pass(self, value: bool) -> None: # leave room for piecewise cuda graph, etc diff --git a/test/srt/test_metrics.py b/test/srt/test_metrics.py index 096a4e221..9fed8451a 100644 --- a/test/srt/test_metrics.py +++ b/test/srt/test_metrics.py @@ -66,6 +66,7 @@ class TestEnableMetrics(CustomTestCase): with ( envs.SGLANG_ENABLE_METRICS_DP_ATTENTION.override(True), envs.SGLANG_ENABLE_METRICS_DEVICE_TIMER.override(True), + envs.SGLANG_TEST_RETRACT.override(True), ): process = popen_launch_server( _MODEL_NAME, @@ -82,12 +83,13 @@ class TestEnableMetrics(CustomTestCase): response = requests.post( f"{DEFAULT_URL_FOR_TEST}/generate", json={ - "text": "The capital of France is", + "text": ["The capital of France is"] * 20, "sampling_params": { "temperature": 0, - "max_new_tokens": 32, + "max_new_tokens": 50, }, "stream": True, + "ignore_eos": True, }, stream=True, )