Fix decode OOM caused by retraction (#14939)
This commit is contained in:
@@ -1609,6 +1609,7 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
|
||||
def retract_decode(
|
||||
self,
|
||||
server_args: ServerArgs,
|
||||
buf_multiplier: int = 1,
|
||||
) -> Tuple[List[Req], float, List[Req]]:
|
||||
"""Retract the decoding requests when there is not enough memory."""
|
||||
sorted_indices = list(range(len(self.reqs)))
|
||||
@@ -1630,7 +1631,9 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
|
||||
retracted_reqs = []
|
||||
first_iter = True
|
||||
while first_iter or (
|
||||
not self.check_decode_mem(selected_indices=sorted_indices)
|
||||
not self.check_decode_mem(
|
||||
selected_indices=sorted_indices, buf_multiplier=buf_multiplier
|
||||
)
|
||||
):
|
||||
if len(sorted_indices) == 1:
|
||||
# Corner case: only one request left
|
||||
|
||||
Reference in New Issue
Block a user