Register allgather/reducescatter buffers with symm memory (#12572)
This commit is contained in:
@@ -263,7 +263,11 @@ class EAGLEDraftCudaGraphRunner:
|
||||
def run_once():
|
||||
# Clean intermediate result cache for DP attention
|
||||
forward_batch.dp_local_start_pos = forward_batch.dp_local_num_tokens = None
|
||||
set_dp_buffer_len(global_dp_buffer_len, num_tokens)
|
||||
set_dp_buffer_len(
|
||||
global_dp_buffer_len,
|
||||
num_tokens,
|
||||
forward_batch.dp_padding_mode.is_max_len(),
|
||||
)
|
||||
set_is_extend_in_batch(False)
|
||||
|
||||
# Backup two fields, which will be modified in-place in `draft_forward`.
|
||||
|
||||
@@ -294,7 +294,11 @@ class EAGLEDraftExtendCudaGraphRunner:
|
||||
def run_once():
|
||||
# Clean intermediate result cache for DP attention
|
||||
forward_batch.dp_local_start_pos = forward_batch.dp_local_num_tokens = None
|
||||
set_dp_buffer_len(global_dp_buffer_len, num_tokens)
|
||||
set_dp_buffer_len(
|
||||
global_dp_buffer_len,
|
||||
num_tokens,
|
||||
forward_batch.dp_padding_mode.is_max_len(),
|
||||
)
|
||||
set_is_extend_in_batch(False)
|
||||
|
||||
# Backup two fields, which will be modified in-place in `draft_forward`.
|
||||
|
||||
Reference in New Issue
Block a user