Fix decode OOM caused by retraction (#14939)

This commit is contained in:
Liangsheng Yin
2025-12-13 13:59:17 +09:00
committed by GitHub
parent 8698867479
commit 01e3b3f3a3
4 changed files with 10 additions and 4 deletions

View File

@@ -1609,6 +1609,7 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
def retract_decode(
self,
server_args: ServerArgs,
buf_multiplier: int = 1,
) -> Tuple[List[Req], float, List[Req]]:
"""Retract the decoding requests when there is not enough memory."""
sorted_indices = list(range(len(self.reqs)))
@@ -1630,7 +1631,9 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
retracted_reqs = []
first_iter = True
while first_iter or (
not self.check_decode_mem(selected_indices=sorted_indices)
not self.check_decode_mem(
selected_indices=sorted_indices, buf_multiplier=buf_multiplier
)
):
if len(sorted_indices) == 1:
# Corner case: only one request left