Clean up __init__ function of the scheduler and event loop for PD (#15298)
This commit is contained in:
@@ -823,20 +823,25 @@ class SchedulerDisaggregationDecodeMixin:
|
||||
"""A normal scheduler loop for decode worker in disaggregation mode."""
|
||||
|
||||
while True:
|
||||
# Receive requests
|
||||
recv_reqs = self.recv_requests()
|
||||
self.process_input_requests(recv_reqs)
|
||||
# polling and allocating kv cache
|
||||
self.process_decode_queue()
|
||||
|
||||
# Get the next batch to run
|
||||
batch = self.get_next_disagg_decode_batch_to_run()
|
||||
self.cur_batch = batch
|
||||
|
||||
# Launch the current batch
|
||||
if batch:
|
||||
# Generate fake extend output.
|
||||
result = self.run_batch(batch)
|
||||
self.process_batch_result(batch, result)
|
||||
else:
|
||||
# When the server is idle, do self-check and re-init some states
|
||||
self.self_check_during_idle()
|
||||
|
||||
# Update last_batch
|
||||
self.last_batch = batch
|
||||
|
||||
@torch.no_grad()
|
||||
@@ -845,26 +850,35 @@ class SchedulerDisaggregationDecodeMixin:
|
||||
self.last_batch: Optional[ScheduleBatch] = None
|
||||
|
||||
while True:
|
||||
|
||||
# Receive requests
|
||||
recv_reqs = self.recv_requests()
|
||||
self.process_input_requests(recv_reqs)
|
||||
# polling and allocating kv cache
|
||||
self.process_decode_queue()
|
||||
|
||||
# Get the next batch to run
|
||||
batch = self.get_next_disagg_decode_batch_to_run()
|
||||
self.cur_batch = batch
|
||||
|
||||
batch_result = None
|
||||
# Launch the current batch
|
||||
if batch:
|
||||
batch_result = self.run_batch(batch)
|
||||
self.result_queue.append((batch.copy(), batch_result))
|
||||
else:
|
||||
batch_result = None
|
||||
|
||||
# Process the last batch
|
||||
if self.last_batch:
|
||||
tmp_batch, tmp_result = self.result_queue.popleft()
|
||||
self.process_batch_result(tmp_batch, tmp_result)
|
||||
elif batch is None:
|
||||
self.self_check_during_idle()
|
||||
|
||||
# Run sample of the current batch
|
||||
# It depends on the result of the last batch (e.g., grammar), so we run it after the last batch is processed.
|
||||
self.launch_batch_sample_if_needed(batch_result)
|
||||
|
||||
# Update last_batch
|
||||
self.last_batch = batch
|
||||
|
||||
def _run_batch_prebuilt(
|
||||
|
||||
@@ -311,6 +311,10 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
def get_next_disagg_prefill_batch_to_run(
|
||||
self: Scheduler,
|
||||
) -> Optional[ScheduleBatch]:
|
||||
# HACK (byronhsu): reset the batch_is_full flag because we never enter update_running_batch which resets it
|
||||
# Otherwise, it hangs under high concurrency
|
||||
self.running_batch.batch_is_full = False
|
||||
|
||||
self.process_prefill_chunk()
|
||||
|
||||
batch = self.get_new_batch_prefill()
|
||||
@@ -327,14 +331,18 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
"""A normal scheduler loop for prefill worker in disaggregation mode."""
|
||||
|
||||
while True:
|
||||
# Receive requests
|
||||
recv_reqs = self.recv_requests()
|
||||
self.process_input_requests(recv_reqs)
|
||||
self.waiting_queue.extend(
|
||||
self.disagg_prefill_bootstrap_queue.pop_bootstrapped()
|
||||
)
|
||||
|
||||
# Get the next batch to run
|
||||
batch = self.get_next_disagg_prefill_batch_to_run()
|
||||
self.cur_batch = batch
|
||||
|
||||
# Launch the current batch
|
||||
if batch:
|
||||
result = self.run_batch(batch)
|
||||
self.process_batch_result_disagg_prefill(batch, result)
|
||||
@@ -343,43 +351,48 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
|
||||
self.process_disagg_prefill_inflight_queue()
|
||||
|
||||
# Update last_batch
|
||||
self.last_batch = batch
|
||||
# HACK (byronhsu): reset the batch_is_full flag because we never enter update_running_batch which resets it
|
||||
# Otherwise, it hangs under high concurrency
|
||||
self.running_batch.batch_is_full = False
|
||||
|
||||
@torch.no_grad()
|
||||
def event_loop_overlap_disagg_prefill(self: Scheduler) -> None:
|
||||
self.result_queue = deque()
|
||||
|
||||
while True:
|
||||
# Receive requests
|
||||
recv_reqs = self.recv_requests()
|
||||
self.process_input_requests(recv_reqs)
|
||||
self.waiting_queue.extend(
|
||||
self.disagg_prefill_bootstrap_queue.pop_bootstrapped()
|
||||
)
|
||||
|
||||
# Get the next batch to run
|
||||
batch = self.get_next_disagg_prefill_batch_to_run()
|
||||
self.cur_batch = batch
|
||||
|
||||
batch_result = None
|
||||
# Launch the current batch
|
||||
if batch:
|
||||
batch_result = self.run_batch(batch)
|
||||
self.result_queue.append((batch.copy(), batch_result))
|
||||
else:
|
||||
batch_result = None
|
||||
|
||||
# Process the last batch
|
||||
if self.last_batch:
|
||||
tmp_batch, tmp_result = self.result_queue.popleft()
|
||||
self.process_batch_result_disagg_prefill(tmp_batch, tmp_result)
|
||||
elif batch is None:
|
||||
# When the server is idle, do self-check and re-init some states
|
||||
self.self_check_during_idle()
|
||||
|
||||
self.process_disagg_prefill_inflight_queue()
|
||||
|
||||
# Run sample of the current batch
|
||||
# It depends on the result of the last batch (e.g., grammar), so we run it after the last batch is processed.
|
||||
self.launch_batch_sample_if_needed(batch_result)
|
||||
|
||||
# Update last_batch
|
||||
self.last_batch = batch
|
||||
# HACK (byronhsu): reset the batch_is_full flag because we never enter update_running_batch which resets it
|
||||
# Otherwise, it hangs under high concurrency
|
||||
self.running_batch.batch_is_full = False
|
||||
|
||||
def process_batch_result_disagg_prefill(
|
||||
self: Scheduler,
|
||||
@@ -420,7 +433,6 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
logits_output.input_token_logprobs.tolist()
|
||||
)
|
||||
|
||||
hidden_state_offset = 0
|
||||
for i, (req, next_token_id) in enumerate(
|
||||
zip(batch.reqs, next_token_ids, strict=True)
|
||||
):
|
||||
|
||||
Reference in New Issue
Block a user