Minor code cleanup / improvement for PREBUILT_EXTEND mode (#12948)

This commit is contained in:
Liangsheng Yin
2025-11-10 11:59:54 +08:00
committed by GitHub
parent 83f89cc615
commit 61c7fe7aed
7 changed files with 50 additions and 55 deletions
+9 -6
View File
@@ -588,7 +588,7 @@ class DecodePreallocQueue:
# the extend batch is not in any queue, so we need to explicitly add the tokens slots here
if (
self.scheduler.last_batch
and self.scheduler.last_batch.forward_mode.is_prebuilt_extend()
and self.scheduler.last_batch.forward_mode.is_prebuilt()
):
allocatable_tokens -= self.num_reserved_decode_tokens * len(
self.scheduler.last_batch.reqs
@@ -850,11 +850,14 @@ class SchedulerDisaggregationDecodeMixin:
self.launch_batch_sample_if_needed(batch_result)
self.last_batch = batch
def _run_batch_prebuilt_extend(
def _run_batch_prebuilt(
self: Scheduler, batch: ScheduleBatch
) -> GenerationBatchResult:
if batch.inner_idle_batch is not None:
return self.run_batch(batch.inner_idle_batch)
idle_batch = batch.inner_idle_batch
# Reset the inner idle batch to avoid reusing it.
batch.inner_idle_batch = None
return self.run_batch(idle_batch)
return GenerationBatchResult()
@@ -864,7 +867,7 @@ class SchedulerDisaggregationDecodeMixin:
"""Create fake completed prefill if possible and merge with running batch"""
# Merge the prefill batch into the running batch
last_batch = self.last_batch
if last_batch and last_batch.forward_mode.is_prebuilt_extend():
if last_batch and last_batch.forward_mode.is_prebuilt():
# chunked prefill doesn't happen in decode instance.
assert self.chunked_req is None
# Filter finished batches.
@@ -947,8 +950,8 @@ class SchedulerDisaggregationDecodeMixin:
)
# construct fake completed prefill
new_batch.prepare_for_prebuilt_extend()
new_batch.process_prebuilt_extend(self.server_args, self.model_config)
new_batch.prepare_for_prebuilt()
new_batch.process_prebuilt(self.server_args, self.model_config)
return new_batch
@@ -20,13 +20,13 @@ if TYPE_CHECKING:
class ScheduleBatchDisaggregationDecodeMixin:
def prepare_for_prebuilt_extend(self: ScheduleBatch):
def prepare_for_prebuilt(self: ScheduleBatch):
"""
Prepare a prebuilt extend by populate metadata
Adapted from .prepare_for_extend().
"""
self.forward_mode = ForwardMode.PREBUILT_EXTEND
self.forward_mode = ForwardMode.PREBUILT
reqs = self.reqs
input_ids = [r.fill_ids[len(r.prefix_indices) :] for r in reqs]
extend_num_tokens = sum(len(ids) for ids in input_ids)
@@ -100,7 +100,7 @@ class ScheduleBatchDisaggregationDecodeMixin:
self.model_config.vocab_size,
)
def process_prebuilt_extend(
def process_prebuilt(
self: ScheduleBatch, server_args: ServerArgs, model_config: ModelConfig
):
"""Assign the buffered last input id to schedule batch"""