chenchenghao/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[misc] Priority scheduling metrics cleanup (#19927)

Browse Source 
Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
sglang-bot
2026-03-05 12:42:42 -08:00
committed by GitHub
parent ff6048fb9c
commit ebb66cc1de
5 changed files with 116 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
python/sglang/srt/managers/scheduler_runtime_checker_mixin.py
View File

@@ -303,30 +303,30 @@ class SchedulerRuntimeCheckerMixin:
else:
num_used, token_usage, _, _ = self._get_token_info()
_enable_ps = self.enable_priority_scheduling
priority_enabled = self.enable_priority_scheduling
self.stats.num_running_reqs = QueueCount.from_reqs(
self.running_batch.reqs, _enable_ps
self.running_batch.reqs, priority_enabled
)
self.stats.num_used_tokens = num_used
self.stats.token_usage = round(token_usage, 2)
self.stats.gen_throughput = 0
self.stats.num_queue_reqs = QueueCount.from_reqs(
self.waiting_queue, _enable_ps
self.waiting_queue, priority_enabled
)
self.stats.num_grammar_queue_reqs = len(self.grammar_manager)
if self.disaggregation_mode == DisaggregationMode.PREFILL:
self.stats.num_prefill_prealloc_queue_reqs = QueueCount.from_reqs(
self.disagg_prefill_bootstrap_queue.queue, _enable_ps
self.disagg_prefill_bootstrap_queue.queue, priority_enabled
)
self.stats.num_prefill_inflight_queue_reqs = QueueCount.from_reqs(
self.disagg_prefill_inflight_queue, _enable_ps
self.disagg_prefill_inflight_queue, priority_enabled
)
if self.disaggregation_mode == DisaggregationMode.DECODE:
self.stats.num_decode_prealloc_queue_reqs = QueueCount.from_reqs(
self.disagg_decode_prealloc_queue.queue, _enable_ps
self.disagg_decode_prealloc_queue.queue, priority_enabled
)
self.stats.num_decode_transfer_queue_reqs = QueueCount.from_reqs(
self.disagg_decode_transfer_queue.queue, _enable_ps
self.disagg_decode_transfer_queue.queue, priority_enabled
)
self.metrics_collector.log_stats(self.stats)
self._publish_kv_events()

45
python/sglang/srt/observability/metrics_collector.py
View File

@@ -758,22 +758,23 @@ class SchedulerMetricsCollector:
multiprocess_mode="mostrecent",
)
def _log_gauge(self, gauge: Gauge, data: Union[int, float, QueueCount]) -> None:
# Convenience function for logging to gauge.
if isinstance(data, QueueCount):
# NOTE: When priority scheduling is enabled, the total is recorded under
# priority="" (the default label value). Per-priority breakdowns are recorded
# with priority="<int>". Grafana queries should use priority="" for totals.
gauge.labels(**self.labels).set(data.total)
if data.by_priority is not None:
self._known_priorities.update(data.by_priority.keys())
for priority in self._known_priorities:
value = data.by_priority.get(priority, 0)
labels = dict(self.labels)
labels["priority"] = str(priority)
gauge.labels(**labels).set(value)
else:
gauge.labels(**self.labels).set(data)
def _log_gauge(self, gauge: Gauge, data: Union[int, float]) -> None:
# Convenience function for logging a scalar to gauge.
gauge.labels(**self.labels).set(data)
def _log_gauge_queue_count(self, gauge: Gauge, data: QueueCount) -> None:
# Log a QueueCount to gauge: total under default labels, per-priority breakdown under priority="<int>".
# NOTE: When priority scheduling is enabled, the total is recorded under
# priority="" (the default label value). Per-priority breakdowns are recorded
# with priority="<int>". Grafana queries should use priority="" for totals.
gauge.labels(**self.labels).set(data.total)
if data.by_priority is not None:
self._known_priorities.update(data.by_priority.keys())
for priority in self._known_priorities:
value = data.by_priority.get(priority, 0)
labels = dict(self.labels)
labels["priority"] = str(priority)
gauge.labels(**labels).set(value)
def _log_histogram(self, histogram, data: Union[int, float]) -> None:
histogram.labels(**self.labels).observe(data)
@@ -900,7 +901,7 @@ class SchedulerMetricsCollector:
).inc(t)
def log_stats(self, stats: SchedulerStats) -> None:
self._log_gauge(self.num_running_reqs, stats.num_running_reqs)
self._log_gauge_queue_count(self.num_running_reqs, stats.num_running_reqs)
self._log_gauge(self.num_used_tokens, stats.num_used_tokens)
self._log_gauge(self.token_usage, stats.token_usage)
self._log_gauge(
@@ -910,7 +911,7 @@ class SchedulerMetricsCollector:
self._log_gauge(self.mamba_usage, stats.mamba_usage)
self._log_gauge(self.decode_sum_seq_lens, stats.decode_sum_seq_lens)
self._log_gauge(self.gen_throughput, stats.gen_throughput)
self._log_gauge(self.num_queue_reqs, stats.num_queue_reqs)
self._log_gauge_queue_count(self.num_queue_reqs, stats.num_queue_reqs)
self._log_gauge(self.num_grammar_queue_reqs, stats.num_grammar_queue_reqs)
self._log_gauge(
self.num_running_reqs_offline_batch, stats.num_running_reqs_offline_batch
@@ -924,16 +925,16 @@ class SchedulerMetricsCollector:
self._log_gauge(self.spec_accept_rate, stats.spec_accept_rate)
# PD disaggregation
self._log_gauge(
self._log_gauge_queue_count(
self.num_prefill_prealloc_queue_reqs, stats.num_prefill_prealloc_queue_reqs
)
self._log_gauge(
self._log_gauge_queue_count(
self.num_prefill_inflight_queue_reqs, stats.num_prefill_inflight_queue_reqs
)
self._log_gauge(
self._log_gauge_queue_count(
self.num_decode_prealloc_queue_reqs, stats.num_decode_prealloc_queue_reqs
)
self._log_gauge(
self._log_gauge_queue_count(
self.num_decode_transfer_queue_reqs, stats.num_decode_transfer_queue_reqs
)
# Retract

28
python/sglang/srt/observability/scheduler_metrics_mixin.py
View File

@@ -283,9 +283,9 @@ class SchedulerMetricsMixin:
if self.is_hybrid_ssm:
self.stats.mamba_usage = mamba_usage
_enable_ps = self.enable_priority_scheduling
priority_enabled = self.enable_priority_scheduling
self.stats.num_queue_reqs = QueueCount.from_reqs(
self.waiting_queue, _enable_ps
self.waiting_queue, priority_enabled
)
self.stats.num_grammar_queue_reqs = len(self.grammar_manager)
self.stats.cache_hit_rate = cache_hit_rate
@@ -300,19 +300,19 @@ class SchedulerMetricsMixin:
# PD disaggregation
if self.disaggregation_mode == DisaggregationMode.PREFILL:
self.stats.num_prefill_prealloc_queue_reqs = QueueCount.from_reqs(
self.disagg_prefill_bootstrap_queue.queue, _enable_ps
self.disagg_prefill_bootstrap_queue.queue, priority_enabled
)
self.stats.num_prefill_inflight_queue_reqs = QueueCount.from_reqs(
self.disagg_prefill_inflight_queue, _enable_ps
self.disagg_prefill_inflight_queue, priority_enabled
)
self.stats.kv_transfer_speed_gb_s = self.kv_transfer_speed_gb_s
self.stats.kv_transfer_latency_ms = self.kv_transfer_latency_ms
elif self.disaggregation_mode == DisaggregationMode.DECODE:
self.stats.num_decode_prealloc_queue_reqs = QueueCount.from_reqs(
self.disagg_decode_prealloc_queue.queue, _enable_ps
self.disagg_decode_prealloc_queue.queue, priority_enabled
)
self.stats.num_decode_transfer_queue_reqs = QueueCount.from_reqs(
self.disagg_decode_transfer_queue.queue, _enable_ps
self.disagg_decode_transfer_queue.queue, priority_enabled
)
# Others
@@ -436,9 +436,11 @@ class SchedulerMetricsMixin:
logger.info(msg)
if self.enable_metrics:
_enable_ps = self.enable_priority_scheduling
priority_enabled = self.enable_priority_scheduling
# Basics
self.stats.num_running_reqs = QueueCount.from_reqs(batch.reqs, _enable_ps)
self.stats.num_running_reqs = QueueCount.from_reqs(
batch.reqs, priority_enabled
)
self.stats.num_running_reqs_offline_batch = num_running_reqs_offline_batch
self.stats.num_used_tokens = num_used
self.stats.token_usage = token_usage
@@ -449,7 +451,7 @@ class SchedulerMetricsMixin:
self.stats.decode_sum_seq_lens = batch.seq_lens_cpu.sum().item()
self.stats.gen_throughput = self.last_gen_throughput
self.stats.num_queue_reqs = QueueCount.from_reqs(
self.waiting_queue, _enable_ps
self.waiting_queue, priority_enabled
)
self.stats.num_grammar_queue_reqs = len(self.grammar_manager)
self.stats.cache_hit_rate = cache_hit_rate
@@ -468,17 +470,17 @@ class SchedulerMetricsMixin:
# PD disaggregation
if self.disaggregation_mode == DisaggregationMode.PREFILL:
self.stats.num_prefill_prealloc_queue_reqs = QueueCount.from_reqs(
self.disagg_prefill_bootstrap_queue.queue, _enable_ps
self.disagg_prefill_bootstrap_queue.queue, priority_enabled
)
self.stats.num_prefill_inflight_queue_reqs = QueueCount.from_reqs(
self.disagg_prefill_inflight_queue, _enable_ps
self.disagg_prefill_inflight_queue, priority_enabled
)
elif self.disaggregation_mode == DisaggregationMode.DECODE:
self.stats.num_decode_prealloc_queue_reqs = QueueCount.from_reqs(
self.disagg_decode_prealloc_queue.queue, _enable_ps
self.disagg_decode_prealloc_queue.queue, priority_enabled
)
self.stats.num_decode_transfer_queue_reqs = QueueCount.from_reqs(
self.disagg_decode_transfer_queue.queue, _enable_ps
self.disagg_decode_transfer_queue.queue, priority_enabled
)
running_routing_keys = [r.routing_key for r in batch.reqs]
waiting_routing_keys = [r.routing_key for r in self.waiting_queue]

15
python/sglang/srt/server_args.py
View File

@@ -5619,6 +5619,21 @@ class ServerArgs:
"fcfs",
"lof",
], f"To use priority scheduling, schedule_policy must be 'fcfs' or 'lof'. '{self.schedule_policy}' is not supported."
if self.default_priority_value is None:
logger.warning(
"--default-priority-value is not set while --enable-priority-scheduling is enabled. "
"Requests without explicit priority will have priority=None, "
"resulting in priority='None' string labels in Prometheus metrics."
)
else:
if self.disable_priority_preemption:
logger.warning(
"--disable-priority-preemption has no effect without --enable-priority-scheduling"
)
if self.default_priority_value is not None:
logger.warning(
"--default-priority-value has no effect without --enable-priority-scheduling"
)
# Check multi-item scoring
if self.multi_item_scoring_delimiter is not None: