Enable native ModelOpt quantization support (2/3) (#9991)
Signed-off-by: Zhiyu Cheng <zhiyuc@nvidia.com>
This commit is contained in:
@@ -178,6 +178,8 @@ class ServerArgs:
|
||||
model_loader_extra_config: str = "{}"
|
||||
trust_remote_code: bool = False
|
||||
modelopt_quant: Optional[Union[str, Dict]] = None
|
||||
modelopt_checkpoint_restore_path: Optional[str] = None
|
||||
modelopt_checkpoint_save_path: Optional[str] = None
|
||||
context_length: Optional[int] = None
|
||||
is_embedding: bool = False
|
||||
enable_multimodal: Optional[bool] = None
|
||||
@@ -1504,6 +1506,21 @@ class ServerArgs:
|
||||
"Supported values: 'fp8', 'int4_awq', 'w4a8_awq', 'nvfp4', 'nvfp4_awq'. "
|
||||
"This requires the NVIDIA Model Optimizer library to be installed: pip install nvidia-modelopt",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--modelopt-checkpoint-restore-path",
|
||||
type=str,
|
||||
default=ServerArgs.modelopt_checkpoint_restore_path,
|
||||
help="Path to restore a previously saved ModelOpt quantized checkpoint. "
|
||||
"If provided, the quantization process will be skipped and the model "
|
||||
"will be loaded from this checkpoint.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--modelopt-checkpoint-save-path",
|
||||
type=str,
|
||||
default=ServerArgs.modelopt_checkpoint_save_path,
|
||||
help="Path to save the ModelOpt quantized checkpoint after quantization. "
|
||||
"This allows reusing the quantized model in future runs.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--kv-cache-dtype",
|
||||
type=str,
|
||||
|
||||
Reference in New Issue
Block a user