Support single batch overlap (#10422)
This commit is contained in:
@@ -377,6 +377,7 @@ class ServerArgs:
|
||||
enable_dp_attention: bool = False
|
||||
enable_dp_lm_head: bool = False
|
||||
enable_two_batch_overlap: bool = False
|
||||
enable_single_batch_overlap: bool = False
|
||||
tbo_token_distribution_threshold: float = 0.48
|
||||
enable_torch_compile: bool = False
|
||||
torch_compile_max_bs: int = 32
|
||||
@@ -2457,6 +2458,11 @@ class ServerArgs:
|
||||
action="store_true",
|
||||
help="Enabling two micro batches to overlap.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--enable-single-batch-overlap",
|
||||
action="store_true",
|
||||
help="Let computation and communication overlap within one micro batch.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--tbo-token-distribution-threshold",
|
||||
type=float,
|
||||
|
||||
Reference in New Issue
Block a user