diff --git a/python/sglang/srt/model_executor/forward_batch_info.py b/python/sglang/srt/model_executor/forward_batch_info.py index e23470dd9..5245ba136 100644 --- a/python/sglang/srt/model_executor/forward_batch_info.py +++ b/python/sglang/srt/model_executor/forward_batch_info.py @@ -360,8 +360,6 @@ class ForwardBatch: batch: ModelWorkerBatch, model_runner: ModelRunner, ): - from sglang.srt.two_batch_overlap import TboForwardBatchPreparer - ret = cls( forward_mode=batch.forward_mode, batch_size=len(batch.seq_lens), @@ -436,9 +434,6 @@ class ForwardBatch: if ret.forward_mode.is_idle(): ret.positions = torch.empty((0,), dtype=torch.int64, device=device) - TboForwardBatchPreparer.prepare( - ret, is_draft_worker=model_runner.is_draft_worker - ) return ret # Override the positions with diffusion LLM or spec_info @@ -496,10 +491,6 @@ class ForwardBatch: if model_runner.server_args.enable_lora: model_runner.lora_manager.prepare_lora_batch(ret) - TboForwardBatchPreparer.prepare( - ret, is_draft_worker=model_runner.is_draft_worker - ) - return ret def merge_mm_inputs(self) -> Optional[MultimodalInputs]: @@ -728,6 +719,8 @@ class ForwardBatch: ) def prepare_mlp_sync_batch(self, model_runner: ModelRunner): + from sglang.srt.two_batch_overlap import TboForwardBatchPreparer + assert self.global_num_tokens_cpu is not None assert self.global_num_tokens_for_logprob_cpu is not None @@ -792,6 +785,8 @@ class ForwardBatch: ) else: bs = self.batch_size = num_tokens + elif self.forward_mode.is_extend(): + self.extend_num_tokens = num_tokens # padding self._pad_inputs_to_size(model_runner, num_tokens, bs) @@ -799,10 +794,15 @@ class ForwardBatch: global_num_tokens_pinned = torch.tensor(global_num_tokens, pin_memory=True) self.global_num_tokens_gpu.copy_(global_num_tokens_pinned, non_blocking=True) + TboForwardBatchPreparer.prepare( + batch=self, is_draft_worker=model_runner.is_draft_worker + ) + def _pad_inputs_to_size(self, model_runner: ModelRunner, num_tokens, bs): # padding self.input_ids = self._pad_tensor_to_size(self.input_ids, num_tokens) self.req_pool_indices = self._pad_tensor_to_size(self.req_pool_indices, bs) + self.lora_ids.extend((bs - len(self.lora_ids)) * [None]) seq_len_fill_value = ( model_runner.attn_backend.get_cuda_graph_seq_len_fill_value()