Improve overlap scheduling (#5788)

This commit is contained in:
Liangsheng Yin
2025-04-28 11:19:16 +08:00
committed by GitHub
parent f0365820e8
commit 40d9b8acce
6 changed files with 61 additions and 23 deletions

View File

@@ -170,13 +170,13 @@ class TpModelWorker:
def forward_batch_generation(
self,
model_worker_batch: ModelWorkerBatch,
launch_done: Optional[threading.Event] = None,
skip_sample: bool = False,
) -> Tuple[LogitsProcessorOutput, Optional[torch.Tensor]]:
forward_batch = ForwardBatch.init_new(model_worker_batch, self.model_runner)
logits_output = self.model_runner.forward(forward_batch)
if launch_done:
launch_done.set()
if model_worker_batch.launch_done is not None:
model_worker_batch.launch_done.set()
if skip_sample:
next_token_ids = None