Minor code cleanup / improvement for PREBUILT_EXTEND mode (#12948)
This commit is contained in:
@@ -588,7 +588,7 @@ class DecodePreallocQueue:
|
||||
# the extend batch is not in any queue, so we need to explicitly add the tokens slots here
|
||||
if (
|
||||
self.scheduler.last_batch
|
||||
and self.scheduler.last_batch.forward_mode.is_prebuilt_extend()
|
||||
and self.scheduler.last_batch.forward_mode.is_prebuilt()
|
||||
):
|
||||
allocatable_tokens -= self.num_reserved_decode_tokens * len(
|
||||
self.scheduler.last_batch.reqs
|
||||
@@ -850,11 +850,14 @@ class SchedulerDisaggregationDecodeMixin:
|
||||
self.launch_batch_sample_if_needed(batch_result)
|
||||
self.last_batch = batch
|
||||
|
||||
def _run_batch_prebuilt_extend(
|
||||
def _run_batch_prebuilt(
|
||||
self: Scheduler, batch: ScheduleBatch
|
||||
) -> GenerationBatchResult:
|
||||
if batch.inner_idle_batch is not None:
|
||||
return self.run_batch(batch.inner_idle_batch)
|
||||
idle_batch = batch.inner_idle_batch
|
||||
# Reset the inner idle batch to avoid reusing it.
|
||||
batch.inner_idle_batch = None
|
||||
return self.run_batch(idle_batch)
|
||||
|
||||
return GenerationBatchResult()
|
||||
|
||||
@@ -864,7 +867,7 @@ class SchedulerDisaggregationDecodeMixin:
|
||||
"""Create fake completed prefill if possible and merge with running batch"""
|
||||
# Merge the prefill batch into the running batch
|
||||
last_batch = self.last_batch
|
||||
if last_batch and last_batch.forward_mode.is_prebuilt_extend():
|
||||
if last_batch and last_batch.forward_mode.is_prebuilt():
|
||||
# chunked prefill doesn't happen in decode instance.
|
||||
assert self.chunked_req is None
|
||||
# Filter finished batches.
|
||||
@@ -947,8 +950,8 @@ class SchedulerDisaggregationDecodeMixin:
|
||||
)
|
||||
|
||||
# construct fake completed prefill
|
||||
new_batch.prepare_for_prebuilt_extend()
|
||||
new_batch.process_prebuilt_extend(self.server_args, self.model_config)
|
||||
new_batch.prepare_for_prebuilt()
|
||||
new_batch.process_prebuilt(self.server_args, self.model_config)
|
||||
|
||||
return new_batch
|
||||
|
||||
|
||||
@@ -20,13 +20,13 @@ if TYPE_CHECKING:
|
||||
|
||||
class ScheduleBatchDisaggregationDecodeMixin:
|
||||
|
||||
def prepare_for_prebuilt_extend(self: ScheduleBatch):
|
||||
def prepare_for_prebuilt(self: ScheduleBatch):
|
||||
"""
|
||||
Prepare a prebuilt extend by populate metadata
|
||||
Adapted from .prepare_for_extend().
|
||||
"""
|
||||
|
||||
self.forward_mode = ForwardMode.PREBUILT_EXTEND
|
||||
self.forward_mode = ForwardMode.PREBUILT
|
||||
reqs = self.reqs
|
||||
input_ids = [r.fill_ids[len(r.prefix_indices) :] for r in reqs]
|
||||
extend_num_tokens = sum(len(ids) for ids in input_ids)
|
||||
@@ -100,7 +100,7 @@ class ScheduleBatchDisaggregationDecodeMixin:
|
||||
self.model_config.vocab_size,
|
||||
)
|
||||
|
||||
def process_prebuilt_extend(
|
||||
def process_prebuilt(
|
||||
self: ScheduleBatch, server_args: ServerArgs, model_config: ModelConfig
|
||||
):
|
||||
"""Assign the buffered last input id to schedule batch"""
|
||||
|
||||
Reference in New Issue
Block a user