Feature/Fix multi lora scheduler blocking issue and evict LoRA None lastly (#14795)
This commit is contained in:
@@ -1817,13 +1817,21 @@ class Scheduler(
|
||||
# Get requests from the waiting queue to a new prefill batch
|
||||
for req in self.waiting_queue:
|
||||
|
||||
if self.enable_lora and not self.tp_worker.can_run_lora_batch(
|
||||
lora_set
|
||||
| set([req.lora_id for req in adder.can_run_list])
|
||||
| set([req.lora_id])
|
||||
):
|
||||
self.running_batch.batch_is_full = True
|
||||
break
|
||||
if self.enable_lora:
|
||||
new_lora_set = (
|
||||
lora_set
|
||||
| set([req.lora_id for req in adder.can_run_list])
|
||||
| set([req.lora_id])
|
||||
)
|
||||
if not self.tp_worker.can_run_lora_batch(new_lora_set):
|
||||
# If this is a LoRA request that would exceed the LoRA slot limit,
|
||||
# skip it and continue to try scheduling non-LoRA requests.
|
||||
# Non-LoRA requests (lora_id=None) share a single reserved slot
|
||||
# and should never cause this check to fail.
|
||||
if req.lora_id is not None:
|
||||
# Skip this LoRA request - it would trigger adapter eviction/loading
|
||||
# which is slow. We'll try to schedule it in a future iteration.
|
||||
continue
|
||||
|
||||
running_bs = len(self.running_batch.reqs)
|
||||
if len(adder.can_run_list) >= self.get_num_allocatable_reqs(running_bs):
|
||||
|
||||
Reference in New Issue
Block a user