Use dynamically maintained num_waiting_tokens in get_load() (#13203)

Signed-off-by: Peng Wang <peng_wang@linux.alibaba.com>
Co-authored-by: Peng Wang <peng_wang@linux.alibaba.com>
This commit is contained in:
vipwangerxiao
2025-11-26 20:34:45 +08:00
committed by GitHub
co-authored by Peng Wang
parent 21b0582d4b
commit 15729dbc8e
3 changed files with 40 additions and 18 deletions
+17 -3
View File
@@ -299,6 +299,11 @@ class DecodePreallocQueue:
if is_retracted:
self.retracted_queue.append(req)
self.scheduler.num_waiting_tokens += (
len(req.origin_input_ids)
+ len(req.output_ids)
+ self.num_reserved_decode_tokens
)
else:
if req.bootstrap_host == FAKE_BOOTSTRAP_HOST:
kv_receiver_class = get_kv_class(
@@ -321,6 +326,7 @@ class DecodePreallocQueue:
self.queue.append(
DecodeRequest(req=req, kv_receiver=kv_receiver, waiting_for_input=False)
)
self.scheduler.num_waiting_tokens += len(req.origin_input_ids)
def _check_if_req_exceed_kv_capacity(self, req: Req) -> bool:
if len(req.origin_input_ids) > self.max_total_num_tokens:
@@ -361,6 +367,8 @@ class DecodePreallocQueue:
req.is_retracted = False
self._pre_alloc(req)
allocatable_tokens -= required_tokens_for_request
# update counter for resumed request
self.scheduler.num_waiting_tokens -= required_tokens_for_request
# load from cpu, release the cpu copy
req.load_kv_cache(self.req_to_token_pool, self.token_to_kv_pool_allocator)
@@ -536,9 +544,15 @@ class DecodePreallocQueue:
RequestStage.DECODE_BOOTSTRAP, decode_req.req.rid, auto_next_anon=True
)
self.queue = [
entry for i, entry in enumerate(self.queue) if i not in indices_to_remove
]
preallocated_queue = []
for i, entry in enumerate(self.queue):
if i not in indices_to_remove:
preallocated_queue.append(entry)
else:
# update counter for allocated or failed request
self.scheduler.num_waiting_tokens -= len(entry.req.origin_input_ids)
self.queue = preallocated_queue
return preallocated_reqs
@@ -200,6 +200,7 @@ class PrefillBootstrapQueue:
self._process_req(req)
req.add_latency(RequestStage.PREFILL_PREPARE)
self.queue.append(req)
self.scheduler.num_waiting_tokens += len(req.origin_input_ids)
trace_slice_end(RequestStage.PREFILL_PREPARE, req.rid, auto_next_anon=True)
def extend(self, reqs: List[Req], num_kv_heads: int) -> None:
@@ -270,6 +271,8 @@ class PrefillBootstrapQueue:
self.scheduler.stream_output([req], req.return_logprob)
indices_to_remove.add(i)
failed_reqs.append(req)
# update counter for failed bootstrap request
self.scheduler.num_waiting_tokens -= len(req.origin_input_ids)
if self.scheduler.enable_metrics:
self.scheduler.metrics_collector.increment_bootstrap_failed_reqs()
continue