Tiny support customizing prometheus buckets for prefill delayer (#16831)

python/sglang/srt/managers/scheduler_metrics_mixin.py

@@ -97,7 +97,7 @@ class SchedulerMetricsMixin:
             self.metrics_collector = SchedulerMetricsCollector(
                 labels=labels,
                 enable_lora=self.enable_lora,
-                prefill_delayer_max_delay_passes=self.server_args.prefill_delayer_max_delay_passes,
+                server_args=self.server_args,
             )
 
             if ENABLE_METRICS_DEVICE_TIMER:

python/sglang/srt/metrics/collector.py

@@ -271,7 +271,7 @@ class SchedulerMetricsCollector:
         self,
         labels: Dict[str, str],
         enable_lora: bool = False,
-        prefill_delayer_max_delay_passes: int = 30,
+        server_args: Optional["ServerArgs"] = None,
     ) -> None:
         # We need to import prometheus_client after setting the env variable `PROMETHEUS_MULTIPROC_DIR`
         from prometheus_client import Counter, Gauge, Histogram, Summary
@@ -762,19 +762,36 @@ class SchedulerMetricsCollector:
             labelnames=list(labels.keys()) + ["category", "num_prefill_ranks"],
         )
 
+        max_delay = server_args.prefill_delayer_max_delay_passes
         self.prefill_delayer_wait_forward_passes = Histogram(
             name="sglang:prefill_delayer_wait_forward_passes",
             documentation="Histogram of forward passes waited by prefill delayer.",
             labelnames=labels.keys(),
-            # Need bucket "<=0" for zero-delay cases
-            buckets=[0, 5, 20, prefill_delayer_max_delay_passes - 1],
+            buckets=sorted(
+                set(
+                    x
+                    for x in (
+                        server_args.prefill_delayer_forward_passes_buckets
+                        or [5, 20, 50, 100, 200]
+                    )
+                    if x < max_delay
+                )
+                # Need bucket "<=0" for zero-delay cases, and "max_delay-1" to distinguish "max_delay" timeout passes
+                | {0, max_delay - 1}
+            ),
         )
         self.prefill_delayer_wait_seconds = Histogram(
             name="sglang:prefill_delayer_wait_seconds",
             documentation="Histogram of wait time in seconds by prefill delayer.",
             labelnames=labels.keys(),
-            # Need bucket "<=0" for zero-delay cases
-            buckets=[0, 5, 20, 100, 500],
+            buckets=sorted(
+                set(
+                    server_args.prefill_delayer_wait_seconds_buckets
+                    or [1, 2, 5, 10, 20, 50, 100, 200, 500]
+                )
+                # Need bucket "<=0" for zero-delay cases
+                | {0}
+            ),
         )
         self.prefill_delayer_outcomes_total = Counter(
             name="sglang:prefill_delayer_outcomes_total",

python/sglang/srt/server_args.py

@@ -314,6 +314,8 @@ class ServerArgs:
     enable_prefill_delayer: bool = False
     prefill_delayer_max_delay_passes: int = 30
     prefill_delayer_token_usage_low_watermark: Optional[float] = None
+    prefill_delayer_forward_passes_buckets: Optional[List[float]] = None
+    prefill_delayer_wait_seconds_buckets: Optional[List[float]] = None
 
     # Runtime options
     device: Optional[str] = None
@@ -2915,6 +2917,20 @@ class ServerArgs:
             default=None,
             help="Token usage low watermark for prefill delayer.",
         )
+        parser.add_argument(
+            "--prefill-delayer-forward-passes-buckets",
+            type=float,
+            nargs="+",
+            default=None,
+            help="Custom buckets for prefill delayer forward passes histogram. 0 and max_delay_passes-1 will be auto-added.",
+        )
+        parser.add_argument(
+            "--prefill-delayer-wait-seconds-buckets",
+            type=float,
+            nargs="+",
+            default=None,
+            help="Custom buckets for prefill delayer wait seconds histogram. 0 will be auto-added.",
+        )
 
         # Runtime options
         parser.add_argument(