Fix get_load API (#13991)
This commit is contained in:
@@ -171,6 +171,10 @@ class DecodeRequest:
|
||||
waiting_for_input: bool = False
|
||||
metadata_buffer_index: int = -1
|
||||
|
||||
@property
|
||||
def seqlen(self) -> int:
|
||||
return self.req.seqlen
|
||||
|
||||
|
||||
class DecodePreallocQueue:
|
||||
"""
|
||||
@@ -299,11 +303,6 @@ class DecodePreallocQueue:
|
||||
|
||||
if is_retracted:
|
||||
self.retracted_queue.append(req)
|
||||
self.scheduler.num_waiting_tokens += (
|
||||
len(req.origin_input_ids)
|
||||
+ len(req.output_ids)
|
||||
+ self.num_reserved_decode_tokens
|
||||
)
|
||||
else:
|
||||
if req.bootstrap_host == FAKE_BOOTSTRAP_HOST:
|
||||
kv_receiver_class = get_kv_class(
|
||||
@@ -326,7 +325,6 @@ class DecodePreallocQueue:
|
||||
self.queue.append(
|
||||
DecodeRequest(req=req, kv_receiver=kv_receiver, waiting_for_input=False)
|
||||
)
|
||||
self.scheduler.num_waiting_tokens += len(req.origin_input_ids)
|
||||
|
||||
def _check_if_req_exceed_kv_capacity(self, req: Req) -> bool:
|
||||
if len(req.origin_input_ids) > self.max_total_num_tokens:
|
||||
@@ -367,8 +365,6 @@ class DecodePreallocQueue:
|
||||
req.is_retracted = False
|
||||
self._pre_alloc(req)
|
||||
allocatable_tokens -= required_tokens_for_request
|
||||
# update counter for resumed request
|
||||
self.scheduler.num_waiting_tokens -= required_tokens_for_request
|
||||
|
||||
# load from cpu, release the cpu copy
|
||||
req.load_kv_cache(self.req_to_token_pool, self.token_to_kv_pool_allocator)
|
||||
@@ -544,15 +540,9 @@ class DecodePreallocQueue:
|
||||
RequestStage.DECODE_BOOTSTRAP, decode_req.req.rid, auto_next_anon=True
|
||||
)
|
||||
|
||||
preallocated_queue = []
|
||||
|
||||
for i, entry in enumerate(self.queue):
|
||||
if i not in indices_to_remove:
|
||||
preallocated_queue.append(entry)
|
||||
else:
|
||||
# update counter for allocated or failed request
|
||||
self.scheduler.num_waiting_tokens -= len(entry.req.origin_input_ids)
|
||||
self.queue = preallocated_queue
|
||||
self.queue = [
|
||||
entry for i, entry in enumerate(self.queue) if i not in indices_to_remove
|
||||
]
|
||||
|
||||
return preallocated_reqs
|
||||
|
||||
|
||||
@@ -200,7 +200,6 @@ class PrefillBootstrapQueue:
|
||||
self._process_req(req)
|
||||
req.add_latency(RequestStage.PREFILL_PREPARE)
|
||||
self.queue.append(req)
|
||||
self.scheduler.num_waiting_tokens += len(req.origin_input_ids)
|
||||
trace_slice_end(RequestStage.PREFILL_PREPARE, req.rid, auto_next_anon=True)
|
||||
|
||||
def extend(self, reqs: List[Req], num_kv_heads: int) -> None:
|
||||
@@ -271,8 +270,6 @@ class PrefillBootstrapQueue:
|
||||
self.scheduler.stream_output([req], req.return_logprob)
|
||||
indices_to_remove.add(i)
|
||||
failed_reqs.append(req)
|
||||
# update counter for failed bootstrap request
|
||||
self.scheduler.num_waiting_tokens -= len(req.origin_input_ids)
|
||||
if self.scheduler.enable_metrics:
|
||||
self.scheduler.metrics_collector.increment_bootstrap_failed_reqs()
|
||||
continue
|
||||
|
||||
Reference in New Issue
Block a user