Add Mistral Large 3 support. (#14213)
Co-authored-by: elvischenv <219235043+elvischenv@users.noreply.github.com> Co-authored-by: Linda-Stadter <57756729+Linda-Stadter@users.noreply.github.com>
This commit is contained in:
co-authored by
elvischenv
Linda-Stadter
parent
af35023e65
commit
8428078436
@@ -58,6 +58,8 @@ def is_deepseek_nsa(config: PretrainedConfig) -> bool:
|
||||
"DeepseekV3ForCausalLM",
|
||||
"DeepseekV32ForCausalLM",
|
||||
"DeepseekV3ForCausalLMNextN",
|
||||
"MistralLarge3ForCausalLM",
|
||||
"PixtralForConditionalGeneration",
|
||||
]
|
||||
and getattr(config, "index_topk", None) is not None
|
||||
)
|
||||
@@ -334,6 +336,8 @@ class ModelConfig:
|
||||
or "LongcatFlashForCausalLM" in self.hf_config.architectures
|
||||
or "LongcatFlashForCausalLMNextN" in self.hf_config.architectures
|
||||
or "DotsVLMForCausalLM" in self.hf_config.architectures
|
||||
or "MistralLarge3ForCausalLM" in self.hf_config.architectures
|
||||
or "PixtralForConditionalGeneration" in self.hf_config.architectures
|
||||
):
|
||||
self.head_dim = 256
|
||||
self.attention_arch = AttentionArch.MLA
|
||||
@@ -939,6 +943,7 @@ multimodal_model_archs = [
|
||||
"MultiModalityCausalLM",
|
||||
"MllamaForConditionalGeneration",
|
||||
"NemotronH_Nano_VL_V2",
|
||||
"PixtralForConditionalGeneration",
|
||||
"Qwen2AudioForConditionalGeneration",
|
||||
"Qwen2VLForConditionalGeneration",
|
||||
"Qwen2_5_VLForConditionalGeneration",
|
||||
|
||||
Reference in New Issue
Block a user