Clean up __init__ function of the scheduler and event loop for PD (#15298)

This commit is contained in:
Lianmin Zheng
2025-12-18 01:35:14 -08:00
committed by GitHub
parent 4792d1f452
commit d1f0063262
9 changed files with 662 additions and 641 deletions
+17 -3
View File
@@ -823,20 +823,25 @@ class SchedulerDisaggregationDecodeMixin:
"""A normal scheduler loop for decode worker in disaggregation mode."""
while True:
# Receive requests
recv_reqs = self.recv_requests()
self.process_input_requests(recv_reqs)
# polling and allocating kv cache
self.process_decode_queue()
# Get the next batch to run
batch = self.get_next_disagg_decode_batch_to_run()
self.cur_batch = batch
# Launch the current batch
if batch:
# Generate fake extend output.
result = self.run_batch(batch)
self.process_batch_result(batch, result)
else:
# When the server is idle, do self-check and re-init some states
self.self_check_during_idle()
# Update last_batch
self.last_batch = batch
@torch.no_grad()
@@ -845,26 +850,35 @@ class SchedulerDisaggregationDecodeMixin:
self.last_batch: Optional[ScheduleBatch] = None
while True:
# Receive requests
recv_reqs = self.recv_requests()
self.process_input_requests(recv_reqs)
# polling and allocating kv cache
self.process_decode_queue()
# Get the next batch to run
batch = self.get_next_disagg_decode_batch_to_run()
self.cur_batch = batch
batch_result = None
# Launch the current batch
if batch:
batch_result = self.run_batch(batch)
self.result_queue.append((batch.copy(), batch_result))
else:
batch_result = None
# Process the last batch
if self.last_batch:
tmp_batch, tmp_result = self.result_queue.popleft()
self.process_batch_result(tmp_batch, tmp_result)
elif batch is None:
self.self_check_during_idle()
# Run sample of the current batch
# It depends on the result of the last batch (e.g., grammar), so we run it after the last batch is processed.
self.launch_batch_sample_if_needed(batch_result)
# Update last_batch
self.last_batch = batch
def _run_batch_prebuilt(
+20 -8
View File
@@ -311,6 +311,10 @@ class SchedulerDisaggregationPrefillMixin:
def get_next_disagg_prefill_batch_to_run(
self: Scheduler,
) -> Optional[ScheduleBatch]:
# HACK (byronhsu): reset the batch_is_full flag because we never enter update_running_batch which resets it
# Otherwise, it hangs under high concurrency
self.running_batch.batch_is_full = False
self.process_prefill_chunk()
batch = self.get_new_batch_prefill()
@@ -327,14 +331,18 @@ class SchedulerDisaggregationPrefillMixin:
"""A normal scheduler loop for prefill worker in disaggregation mode."""
while True:
# Receive requests
recv_reqs = self.recv_requests()
self.process_input_requests(recv_reqs)
self.waiting_queue.extend(
self.disagg_prefill_bootstrap_queue.pop_bootstrapped()
)
# Get the next batch to run
batch = self.get_next_disagg_prefill_batch_to_run()
self.cur_batch = batch
# Launch the current batch
if batch:
result = self.run_batch(batch)
self.process_batch_result_disagg_prefill(batch, result)
@@ -343,43 +351,48 @@ class SchedulerDisaggregationPrefillMixin:
self.process_disagg_prefill_inflight_queue()
# Update last_batch
self.last_batch = batch
# HACK (byronhsu): reset the batch_is_full flag because we never enter update_running_batch which resets it
# Otherwise, it hangs under high concurrency
self.running_batch.batch_is_full = False
@torch.no_grad()
def event_loop_overlap_disagg_prefill(self: Scheduler) -> None:
self.result_queue = deque()
while True:
# Receive requests
recv_reqs = self.recv_requests()
self.process_input_requests(recv_reqs)
self.waiting_queue.extend(
self.disagg_prefill_bootstrap_queue.pop_bootstrapped()
)
# Get the next batch to run
batch = self.get_next_disagg_prefill_batch_to_run()
self.cur_batch = batch
batch_result = None
# Launch the current batch
if batch:
batch_result = self.run_batch(batch)
self.result_queue.append((batch.copy(), batch_result))
else:
batch_result = None
# Process the last batch
if self.last_batch:
tmp_batch, tmp_result = self.result_queue.popleft()
self.process_batch_result_disagg_prefill(tmp_batch, tmp_result)
elif batch is None:
# When the server is idle, do self-check and re-init some states
self.self_check_during_idle()
self.process_disagg_prefill_inflight_queue()
# Run sample of the current batch
# It depends on the result of the last batch (e.g., grammar), so we run it after the last batch is processed.
self.launch_batch_sample_if_needed(batch_result)
# Update last_batch
self.last_batch = batch
# HACK (byronhsu): reset the batch_is_full flag because we never enter update_running_batch which resets it
# Otherwise, it hangs under high concurrency
self.running_batch.batch_is_full = False
def process_batch_result_disagg_prefill(
self: Scheduler,
@@ -420,7 +433,6 @@ class SchedulerDisaggregationPrefillMixin:
logits_output.input_token_logprobs.tolist()
)
hidden_state_offset = 0
for i, (req, next_token_id) in enumerate(
zip(batch.reqs, next_token_ids, strict=True)
):