diff --git a/python/sglang/srt/managers/schedule_batch.py b/python/sglang/srt/managers/schedule_batch.py index 2ff3b406d..e156950d9 100644 --- a/python/sglang/srt/managers/schedule_batch.py +++ b/python/sglang/srt/managers/schedule_batch.py @@ -1891,12 +1891,23 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin): # release memory and don't insert into the tree because we need the space instantly self.release_req(idx, len(sorted_indices), server_args) + reqs_to_abort: List[Req] = [] if len(sorted_indices) <= 1 and not self.check_decode_mem( selected_indices=sorted_indices ): - # Retracting loops ends and still not enough memory - raise ValueError( - "Out of memory even after retracting all other requests in the decode batch." + # Even the last remaining request cannot fit in memory. + # Instead of crashing the scheduler, gracefully abort it. + last_idx = sorted_indices.pop() + last_req = self.reqs[last_idx] + last_req.to_finish = FINISH_ABORT( + "Out of memory even after retracting all other requests " + "in the decode batch. Aborting the last request.", + status_code=HTTPStatus.INTERNAL_SERVER_ERROR, + ) + reqs_to_abort.append(last_req) + self.release_req(last_idx, 0, server_args) + logger.warning( + "retract_decode: aborted last request %s due to OOM", last_req.rid ) self.filter_batch(keep_indices=sorted_indices) @@ -1913,7 +1924,7 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin): ) # avoid zero division new_estimate_ratio = min(1.0, new_estimate_ratio) - return retracted_reqs, new_estimate_ratio, [] + return retracted_reqs, new_estimate_ratio, reqs_to_abort def release_req(self, idx: int, remaing_req_count: int, server_args: ServerArgs): req = self.reqs[idx] diff --git a/python/sglang/srt/managers/scheduler.py b/python/sglang/srt/managers/scheduler.py index 2a4f350dd..acd507277 100644 --- a/python/sglang/srt/managers/scheduler.py +++ b/python/sglang/srt/managers/scheduler.py @@ -2387,7 +2387,11 @@ class Scheduler( for req in reqs_to_abort: abort_reason: FINISH_ABORT = req.to_finish self.send_to_tokenizer.send_output( - AbortReq(abort_message=abort_reason.message, rid=req.rid), req + AbortReq( + finished_reason=abort_reason.to_json(), + rid=req.rid, + ), + req, ) msg_prefix = ( @@ -2413,6 +2417,9 @@ class Scheduler( if batch.batch_size() < initial_bs: batch.batch_is_full = False + if batch.is_empty(): + return batch + # Update batch tensors batch.prepare_for_decode() return batch