Improve the metrics for PD (#12580)
Co-authored-by: Kan Wu <wukanustc@gmail.com> Co-authored-by: cctry <shiyang@x.ai>
This commit is contained in:
@@ -552,6 +552,10 @@ class ServerArgs:
|
||||
mm_max_concurrent_calls: int = 32
|
||||
mm_per_request_timeout: float = 10.0
|
||||
|
||||
# For checkpoint decryption
|
||||
decrypted_config_file: Optional[str] = None
|
||||
decrypted_draft_config_file: Optional[str] = None
|
||||
|
||||
def __post_init__(self):
|
||||
"""
|
||||
Orchestrates the handling of various server arguments, ensuring proper configuration and validation.
|
||||
@@ -3551,6 +3555,20 @@ class ServerArgs:
|
||||
help="The timeout for each multi-modal request in seconds.",
|
||||
)
|
||||
|
||||
# For checkpoint decryption
|
||||
parser.add_argument(
|
||||
"--decrypted-config-file",
|
||||
type=str,
|
||||
default=ServerArgs.decrypted_config_file,
|
||||
help="The path of the decrypted config file.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--decrypted-draft-config-file",
|
||||
type=str,
|
||||
default=ServerArgs.decrypted_draft_config_file,
|
||||
help="The path of the decrypted draft config file.",
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_cli_args(cls, args: argparse.Namespace):
|
||||
args.tp_size = args.tensor_parallel_size
|
||||
|
||||
Reference in New Issue
Block a user