From 252ef90fc2ea8dad02ef571279313e3753a3d06a Mon Sep 17 00:00:00 2001 From: haNa-meister <34223424+haNa-meister@users.noreply.github.com> Date: Wed, 11 Mar 2026 13:15:50 -0700 Subject: [PATCH] [Generative Score API] Fix on prefill-only scheduler running batch loss track problem (#14320) Co-authored-by: Wenyan Yao Co-authored-by: Sundara Raman Ramachandran --- python/sglang/srt/managers/scheduler.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/python/sglang/srt/managers/scheduler.py b/python/sglang/srt/managers/scheduler.py index 53457c1c1..2a4f350dd 100644 --- a/python/sglang/srt/managers/scheduler.py +++ b/python/sglang/srt/managers/scheduler.py @@ -2059,14 +2059,21 @@ class Scheduler( self.running_batch.batch_is_full = False # Merge the new batch into the running batch. - # For prefill-only batch, we can avoid going through decoding step. - if not self.last_batch.is_empty() and not self.last_batch.is_prefill_only: + if not self.last_batch.is_empty(): if self.running_batch.is_empty(): self.running_batch = self.last_batch else: # Merge running_batch with prefill batch self.running_batch.merge_batch(self.last_batch) + # For prefill-only batch, filter out finished requests since they + # won't go through the decode step. This keeps running_batch accurate + # for load reporting (num_running_reqs via /get_load). + # Runs outside the last_batch block so stale requests are cleaned + # even when no new batches arrive (e.g. traffic stops). + if self.running_batch.is_prefill_only: + self.running_batch.filter_batch() + if self.dllm_config is not None: new_batch = self.get_new_batch_dllm() else: @@ -2085,8 +2092,11 @@ class Scheduler( # Run prefill first if possible ret = new_batch else: - # Run decode - if not self.running_batch.is_empty(): + # Run decode (skip for prefill-only batches) + if ( + not self.running_batch.is_empty() + and not self.running_batch.is_prefill_only + ): self.running_batch = self.update_running_batch(self.running_batch) ret = self.running_batch if not self.running_batch.is_empty() else None else: