Deepseek v2 support (#693)

2024-07-26 17:10:07 -07:00
parent 5bd06b4599
commit 679ebcbbdc
5 changed files with 538 additions and 6 deletions
--- a/python/sglang/srt/model_config.py
+++ b/python/sglang/srt/model_config.py
@@ -36,6 +36,11 @@ class ModelConfig:
            "head_dim",
            self.hf_config.hidden_size // self.hf_config.num_attention_heads,
        )
+
+        # FIXME: temporary special judge for deepseek v2 MLA architecture
+        if "DeepseekV2ForCausalLM" in self.hf_config.architectures:
+            self.head_dim = 256
+
        self.num_attention_heads = self.hf_config.num_attention_heads
        self.num_key_value_heads = getattr(self.hf_config, "num_key_value_heads", None)