chenchenghao/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 14 Packages Projects Releases Wiki Activity

Improve the metrics for PD (#12580)

Browse Source 
Co-authored-by: Kan Wu <wukanustc@gmail.com>
Co-authored-by: cctry <shiyang@x.ai>
This commit is contained in:
Lianmin Zheng
2025-11-03 22:10:57 -08:00
committed by GitHub
co-authored by Kan Wu cctry
parent 93be7e863e
commit f600866a44
5 changed files with 79 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/sglang/srt/metrics/collector.py
+48 -2
View File
@@ -43,6 +43,9 @@ class TimeStats:
prefill_transfer_queue_entry_time: float = 0.0
decode_prealloc_queue_entry_time: float = 0.0
decode_transfer_queue_entry_time: float = 0.0
# TODO: correct set them
bootstrap_duration: float = 0.0
alloc_waiting_duration: float = 0.0
def get_queueing_time(self) -> float:
return self.forward_entry_time - self.wait_queue_entry_time
@@ -73,7 +76,20 @@ class TimeStats:
and forward_duration >= 0
), f"bootstrap_duration={bootstrap_duration} < 0 or queue_duration={queue_duration} < 0 or forward_duration={forward_duration} < 0"
return f"bootstrap_duration={self.format_duration(bootstrap_duration)}, queue_duration={self.format_duration(queue_duration)}, forward_duration={self.format_duration(forward_duration)}, start_time={self.prefill_bootstrap_queue_entry_time:.3f}"
other = max(
0.0,
bootstrap_duration
- (self.alloc_waiting_duration + self.bootstrap_duration),
)
return (
f"bootstrap_queue_duration({self.format_duration(bootstrap_duration)}) "
f"= alloc_wait({self.format_duration(self.alloc_waiting_duration)}) "
f"+ bootstrap({self.format_duration(self.bootstrap_duration)}) "
f"+ other({self.format_duration(other)}); "
f"queue_duration={self.format_duration(queue_duration)}, "
f"forward_duration={self.format_duration(forward_duration)}, "
f"start={self.prefill_bootstrap_queue_entry_time:.3f}"
)
elif self.disagg_mode == DisaggregationMode.DECODE:
prealloc_duration = (
self.decode_transfer_queue_entry_time
@@ -94,7 +110,21 @@ class TimeStats:
and forward_duration >= 0
), f"prealloc_duration={prealloc_duration} < 0 or transfer_duration={transfer_duration} < 0 or queue_duration={queue_duration} < 0 or forward_duration={forward_duration} < 0. {self=}"
return f"prealloc_duration={self.format_duration(prealloc_duration)}, transfer_duration={self.format_duration(transfer_duration)}, queue_duration={self.format_duration(queue_duration)}, forward_duration={self.format_duration(forward_duration)}, start_time={self.decode_prealloc_queue_entry_time:.3f}"
other = max(
0.0,
prealloc_duration
- (self.alloc_waiting_duration + self.bootstrap_duration),
)
return (
f"prealloc_queue_duration({self.format_duration(prealloc_duration)}) "
f"= alloc_wait({self.format_duration(self.alloc_waiting_duration)}) "
f"+ bootstrap({self.format_duration(self.bootstrap_duration)}) "
f"+ other({self.format_duration(other)}); "
f"transfer_duration={self.format_duration(transfer_duration)}; "
f"queue_duration={self.format_duration(queue_duration)}, "
f"forward_duration={self.format_duration(forward_duration)}, "
f"start={self.decode_prealloc_queue_entry_time:.3f}"
)
else:
return "Unknown Time Stats"
@@ -141,6 +171,8 @@ class SchedulerStats:
num_decode_transfer_queue_reqs: int = 0
kv_transfer_speed_gb_s: float = 0.0
kv_transfer_latency_ms: float = 0.0
kv_transfer_bootstrap_ms: float = 0.0
kv_transfer_alloc_ms: float = 0.0
# Utilization
utilization: float = 0.0
@@ -297,6 +329,18 @@ class SchedulerMetricsCollector:
labelnames=labels.keys(),
multiprocess_mode="mostrecent",
)
self.kv_transfer_bootstrap_ms = Gauge(
name="sglang:kv_transfer_bootstrap_ms",
documentation="The bootstrap time of the KV transfer in ms.",
labelnames=labels.keys(),
multiprocess_mode="mostrecent",
)
self.kv_transfer_alloc_ms = Gauge(
name="sglang:kv_transfer_alloc_ms",
documentation="The allocation waiting time of the KV transfer in ms.",
labelnames=labels.keys(),
multiprocess_mode="mostrecent",
)
# Utilization
self.utilization = Gauge(
@@ -564,6 +608,8 @@ class SchedulerMetricsCollector:
)
self._log_gauge(self.kv_transfer_speed_gb_s, stats.kv_transfer_speed_gb_s)
self._log_gauge(self.kv_transfer_latency_ms, stats.kv_transfer_latency_ms)
self._log_gauge(self.kv_transfer_bootstrap_ms, stats.kv_transfer_bootstrap_ms)
self._log_gauge(self.kv_transfer_alloc_ms, stats.kv_transfer_alloc_ms)
# Retract
self._log_gauge(self.num_retracted_reqs, stats.num_retracted_reqs)