Add Mistral Large 3 Eagle Support (#14466)
Co-authored-by: Linda-Stadter <57756729+Linda-Stadter@users.noreply.github.com>
This commit is contained in:
co-authored by
Linda-Stadter
parent
7235a7fbe9
commit
205f041e96
@@ -22,11 +22,15 @@ def adapt_config_dict(
|
||||
is_mistral_large_3 = (
|
||||
is_moe and (config_dict["moe"].get("num_shared_experts") or 0) > 0
|
||||
)
|
||||
is_eagle = "eagle" in model.lower()
|
||||
if is_moe:
|
||||
if is_mistral_large_3:
|
||||
config_dict = _remap_moe_args(config_dict)
|
||||
config_dict["model_type"] = "deepseek_v3"
|
||||
config_dict["architectures"] = ["MistralLarge3ForCausalLM"]
|
||||
if is_eagle:
|
||||
config_dict["architectures"] = ["MistralLarge3ForCausalLMEagle"]
|
||||
else:
|
||||
config_dict["architectures"] = ["MistralLarge3ForCausalLM"]
|
||||
|
||||
assert (
|
||||
"llama_4_scaling" in config_dict
|
||||
@@ -77,6 +81,8 @@ def adapt_config_dict(
|
||||
config_dict = _remap_mistral_vision_args(config_dict)
|
||||
if is_audio:
|
||||
config_dict = _remap_mistral_audio_args(config_dict)
|
||||
if is_eagle:
|
||||
config_dict["routing_method_type"] = 1 # RoutingMethodType.Renormalize
|
||||
|
||||
config = PretrainedConfig.from_dict(config_dict)
|
||||
|
||||
@@ -227,7 +233,6 @@ def _remap_moe_args(config: dict) -> dict:
|
||||
config[new_name] = value
|
||||
|
||||
config["topk_method"] = None
|
||||
config["routing_method_type"] = 1 # RoutingMethodType.Renormalize
|
||||
config["scoring_func"] = "softmax"
|
||||
|
||||
return config
|
||||
|
||||
Reference in New Issue
Block a user