support llama4 eagle3 (#6985)

Co-authored-by: shuaills <shishuaiuoe@gmail.com>
Co-authored-by: Shenggui Li <somerlee.9@gmail.com>
Co-authored-by: Yingyi Huang <yingyihuang2000@outlook.com>
Co-authored-by: yizhang2077 <1109276519@qq.com>
This commit is contained in:
lukec
2025-06-30 22:34:10 -07:00
committed by GitHub
co-authored by shuaills Shenggui Li Yingyi Huang yizhang2077
parent 637bfee448
commit 886d344964
7 changed files with 138 additions and 18 deletions
@@ -209,6 +209,17 @@ def get_quant_config(
config["adapter_name_or_path"] = model_name_or_path
elif model_config.quantization == "modelopt":
if config["producer"]["name"] == "modelopt":
# (yizhang2077) workaround for nvidia/Llama-4-Maverick-17B-128E-Eagle3
if config["quantization"]["quant_algo"] is None:
if (
model_config.hf_config.architectures[0]
!= "LlamaForCausalLMEagle3"
):
raise ValueError(
f"Invalid quant_config, quantization method: {model_config.quantization},"
f"hf architectures: {model_config.hf_config.architectures[0]}. "
)
return None
if "FP4" in config["quantization"]["quant_algo"]:
return ModelOptFp4Config.from_config(config)
else: