Fix spec info's filter when reqs are finished right after prefill (#14742)
This commit is contained in:
@@ -1937,7 +1937,7 @@ class Scheduler(
|
||||
and not (new_batch.return_logprob or self.running_batch.return_logprob)
|
||||
):
|
||||
# TODO (lianmin): support return_logprob + mixed chunked prefill
|
||||
self.running_batch.filter_batch()
|
||||
self.running_batch.filter_batch(v1_spec_info_filtered=True)
|
||||
if not self.running_batch.is_empty():
|
||||
self.running_batch.prepare_for_decode()
|
||||
new_batch.mix_with_running(self.running_batch)
|
||||
@@ -1954,7 +1954,7 @@ class Scheduler(
|
||||
"""Update the current running decoding batch."""
|
||||
initial_bs = batch.batch_size()
|
||||
|
||||
batch.filter_batch()
|
||||
batch.filter_batch(v1_spec_info_filtered=True)
|
||||
if batch.is_empty():
|
||||
batch.batch_is_full = False
|
||||
return batch
|
||||
@@ -2509,7 +2509,7 @@ class Scheduler(
|
||||
self.cur_batch = None
|
||||
|
||||
if recv_req.mode == "retract":
|
||||
self.running_batch.filter_batch()
|
||||
self.running_batch.filter_batch(v1_spec_info_filtered=True)
|
||||
if len(self.running_batch.reqs) != 0:
|
||||
retracted_reqs = self.running_batch.retract_all(self.server_args)
|
||||
for req in retracted_reqs:
|
||||
|
||||
Reference in New Issue
Block a user