Support Kimi Linear (#12469)

Co-authored-by: yizhang2077 <1109276519@qq.com>
2025-10-31 14:03:35 -07:00
parent 30ad107028
commit a4bf5c6ad2
18 changed files with 2847 additions and 112 deletions
@@ -366,6 +366,13 @@ class ModelConfig:
            self.qk_rope_head_dim = self.hf_text_config.qk_rope_head_dim
            self.v_head_dim = self.hf_text_config.v_head_dim
            self.qk_nope_head_dim = self.hf_text_config.qk_nope_head_dim
+        elif "KimiLinearForCausalLM" in self.hf_config.architectures:
+            self.head_dim = 72
+            self.attention_arch = AttentionArch.MLA
+            self.kv_lora_rank = self.hf_config.kv_lora_rank
+            self.qk_rope_head_dim = self.hf_config.qk_rope_head_dim
+            self.v_head_dim = self.hf_config.v_head_dim
+            self.qk_nope_head_dim = self.hf_config.qk_nope_head_dim
        else:
            if (
                "MistralModel" in self.hf_config.architectures