Support single batch overlap (#10422)

This commit is contained in:
fzyzcjy
2025-10-02 18:04:36 +08:00
committed by GitHub
parent 0b9dfba787
commit 5e786cca3a
9 changed files with 268 additions and 20 deletions
+6
View File
@@ -377,6 +377,7 @@ class ServerArgs:
enable_dp_attention: bool = False
enable_dp_lm_head: bool = False
enable_two_batch_overlap: bool = False
enable_single_batch_overlap: bool = False
tbo_token_distribution_threshold: float = 0.48
enable_torch_compile: bool = False
torch_compile_max_bs: int = 32
@@ -2457,6 +2458,11 @@ class ServerArgs:
action="store_true",
help="Enabling two micro batches to overlap.",
)
parser.add_argument(
"--enable-single-batch-overlap",
action="store_true",
help="Let computation and communication overlap within one micro batch.",
)
parser.add_argument(
"--tbo-token-distribution-threshold",
type=float,