Feature/Fix multi lora scheduler blocking issue and evict LoRA None lastly (#14795)

This commit is contained in:
Chenxi Li
2025-12-12 17:13:05 -08:00
committed by GitHub
parent 44fd701732
commit 9b9d21312a
3 changed files with 45 additions and 26 deletions
+15 -7
View File
@@ -1817,13 +1817,21 @@ class Scheduler(
# Get requests from the waiting queue to a new prefill batch
for req in self.waiting_queue:
if self.enable_lora and not self.tp_worker.can_run_lora_batch(
lora_set
| set([req.lora_id for req in adder.can_run_list])
| set([req.lora_id])
):
self.running_batch.batch_is_full = True
break
if self.enable_lora:
new_lora_set = (
lora_set
| set([req.lora_id for req in adder.can_run_list])
| set([req.lora_id])
)
if not self.tp_worker.can_run_lora_batch(new_lora_set):
# If this is a LoRA request that would exceed the LoRA slot limit,
# skip it and continue to try scheduling non-LoRA requests.
# Non-LoRA requests (lora_id=None) share a single reserved slot
# and should never cause this check to fail.
if req.lora_id is not None:
# Skip this LoRA request - it would trigger adapter eviction/loading
# which is slow. We'll try to schedule it in a future iteration.
continue
running_bs = len(self.running_batch.reqs)
if len(adder.can_run_list) >= self.get_num_allocatable_reqs(running_bs):