Handle abort for retracted requests in disagg decode prealloc queue (#18705)
Co-authored-by: sunhailiang <sunhailiang@baidu.com> Co-authored-by: Liangsheng Yin <lsyincs@gmail.com>
This commit is contained in:
@@ -2817,6 +2817,20 @@ class Scheduler(
|
||||
logger.debug(f"Abort transfer queue request. {decode_req.req.rid=}")
|
||||
decode_req.kv_receiver.abort()
|
||||
|
||||
# Abort requests already retracted to CPU cache
|
||||
if self.disagg_decode_prealloc_queue.retracted_queue:
|
||||
remaining_retracted = []
|
||||
for decode_req in self.disagg_decode_prealloc_queue.retracted_queue:
|
||||
if recv_req.abort_all or decode_req.rid.startswith(recv_req.rid):
|
||||
assert hasattr(decode_req, "kv_cache_cpu")
|
||||
del decode_req.kv_cache_cpu
|
||||
self.send_to_tokenizer.send_output(
|
||||
AbortReq(rid=decode_req.rid), decode_req
|
||||
)
|
||||
else:
|
||||
remaining_retracted.append(decode_req)
|
||||
self.disagg_decode_prealloc_queue.retracted_queue = remaining_retracted
|
||||
|
||||
# Delete requests in the running batch
|
||||
if self.cur_batch is self.running_batch or self.cur_batch is None:
|
||||
reqs = self.running_batch.reqs
|
||||
|
||||
Reference in New Issue
Block a user