Enable native ModelOpt quantization support (3/3) (#10154)

Signed-off-by: Zhiyu Cheng <zhiyuc@nvidia.com>
This commit is contained in:
Zhiyu
2025-10-21 21:44:29 -07:00
committed by GitHub
parent 4b65ed42cc
commit 80b2b3207a
16 changed files with 1528 additions and 39 deletions

View File

@@ -83,6 +83,7 @@ QUANTIZATION_CHOICES = [
"bitsandbytes",
"gguf",
"modelopt",
"modelopt_fp8",
"modelopt_fp4",
"petit_nvfp4",
"w8a8_int8",
@@ -192,6 +193,8 @@ class ServerArgs:
modelopt_quant: Optional[Union[str, Dict]] = None
modelopt_checkpoint_restore_path: Optional[str] = None
modelopt_checkpoint_save_path: Optional[str] = None
modelopt_export_path: Optional[str] = None
quantize_and_serve: bool = False
context_length: Optional[int] = None
is_embedding: bool = False
enable_multimodal: Optional[bool] = None
@@ -1743,6 +1746,22 @@ class ServerArgs:
help="Path to save the ModelOpt quantized checkpoint after quantization. "
"This allows reusing the quantized model in future runs.",
)
parser.add_argument(
"--modelopt-export-path",
type=str,
default=ServerArgs.modelopt_export_path,
help="Path to export the quantized model in HuggingFace format after ModelOpt quantization. "
"The exported model can then be used directly with SGLang for inference. "
"If not provided, the model will not be exported.",
)
parser.add_argument(
"--quantize-and-serve",
action="store_true",
default=ServerArgs.quantize_and_serve,
help="Quantize the model with ModelOpt and immediately serve it without exporting. "
"This is useful for development and prototyping. For production, it's recommended "
"to use separate quantization and deployment steps.",
)
parser.add_argument(
"--kv-cache-dtype",
type=str,