Move unnecessary input_addr capture under debug mode flag for speed-up (#13690)

This commit is contained in:
Binyao Jiang
2025-11-22 11:42:26 -08:00
committed by GitHub
parent 3990b84bd3
commit b29769f3b6
5 changed files with 24 additions and 8 deletions

View File

@@ -501,6 +501,7 @@ class ServerArgs:
tbo_token_distribution_threshold: float = 0.48
enable_torch_compile: bool = False
enable_piecewise_cuda_graph: bool = False
enable_torch_compile_debug_mode: bool = False
torch_compile_max_bs: int = 32
piecewise_cuda_graph_max_tokens: int = 4096
piecewise_cuda_graph_tokens: Optional[List[int]] = None
@@ -3413,6 +3414,11 @@ class ServerArgs:
action="store_true",
help="Optimize the model with torch.compile. Experimental feature.",
)
parser.add_argument(
"--enable-torch-compile-debug-mode",
action="store_true",
help="Enable debug mode for torch compile",
)
parser.add_argument(
"--enable-piecewise-cuda-graph",
action="store_true",