Refactor / Unify event loop across PD-Disagg, Overlap, DP-Attn cases (#12839)
Co-authored-by: cctry <17473714+cctry@users.noreply.github.com>
This commit is contained in:
@@ -68,7 +68,7 @@ def get_token_num_per_seq(
|
||||
|
||||
# TODO: may smartly disable TBO when batch size is too small b/c it will slow down
|
||||
def compute_split_seq_index(
|
||||
forward_mode: "ForwardMode",
|
||||
forward_mode: ForwardMode,
|
||||
num_tokens: int,
|
||||
extend_lens: Optional[Sequence[int]],
|
||||
token_num_per_seq: Optional[int],
|
||||
@@ -79,7 +79,7 @@ def compute_split_seq_index(
|
||||
elif forward_mode.is_target_verify() or forward_mode.is_decode():
|
||||
assert token_num_per_seq is not None
|
||||
return (num_tokens // token_num_per_seq) // 2
|
||||
elif forward_mode.is_idle():
|
||||
elif forward_mode.is_idle() or forward_mode.is_prebuilt_extend():
|
||||
assert num_tokens == 0
|
||||
return 0
|
||||
else:
|
||||
@@ -381,6 +381,8 @@ class TboDPAttentionPreparer:
|
||||
or local_batch.forward_mode.is_decode()
|
||||
):
|
||||
num_tokens = local_batch.batch_size() * token_num_per_seq
|
||||
elif local_batch.forward_mode.is_prebuilt_extend():
|
||||
num_tokens = 0
|
||||
else:
|
||||
num_tokens = local_batch.extend_num_tokens
|
||||
self.local_tbo_split_seq_index = compute_split_seq_index(
|
||||
|
||||
Reference in New Issue
Block a user