Add Mistral Large 3 Eagle Support (#14466)

Co-authored-by: Linda-Stadter <57756729+Linda-Stadter@users.noreply.github.com>
This commit is contained in:
elvischenv
2025-12-05 23:11:41 +08:00
committed by GitHub
co-authored by Linda-Stadter
parent 7235a7fbe9
commit 205f041e96
9 changed files with 313 additions and 62 deletions
+7 -2
View File
@@ -22,11 +22,15 @@ def adapt_config_dict(
is_mistral_large_3 = (
is_moe and (config_dict["moe"].get("num_shared_experts") or 0) > 0
)
is_eagle = "eagle" in model.lower()
if is_moe:
if is_mistral_large_3:
config_dict = _remap_moe_args(config_dict)
config_dict["model_type"] = "deepseek_v3"
config_dict["architectures"] = ["MistralLarge3ForCausalLM"]
if is_eagle:
config_dict["architectures"] = ["MistralLarge3ForCausalLMEagle"]
else:
config_dict["architectures"] = ["MistralLarge3ForCausalLM"]
assert (
"llama_4_scaling" in config_dict
@@ -77,6 +81,8 @@ def adapt_config_dict(
config_dict = _remap_mistral_vision_args(config_dict)
if is_audio:
config_dict = _remap_mistral_audio_args(config_dict)
if is_eagle:
config_dict["routing_method_type"] = 1 # RoutingMethodType.Renormalize
config = PretrainedConfig.from_dict(config_dict)
@@ -227,7 +233,6 @@ def _remap_moe_args(config: dict) -> dict:
config[new_name] = value
config["topk_method"] = None
config["routing_method_type"] = 1 # RoutingMethodType.Renormalize
config["scoring_func"] = "softmax"
return config