Add Mistral Large 3 support. (#14213)
Co-authored-by: elvischenv <219235043+elvischenv@users.noreply.github.com> Co-authored-by: Linda-Stadter <57756729+Linda-Stadter@users.noreply.github.com>
This commit is contained in:
@@ -944,7 +944,18 @@ class ServerArgs:
|
||||
|
||||
hf_config = self.get_hf_config()
|
||||
model_arch = hf_config.architectures[0]
|
||||
if model_arch in ["DeepseekV3ForCausalLM"]:
|
||||
|
||||
if model_arch in [
|
||||
"MistralLarge3ForCausalLM",
|
||||
"PixtralForConditionalGeneration",
|
||||
]:
|
||||
self.dtype = "bfloat16"
|
||||
|
||||
if model_arch in [
|
||||
"DeepseekV3ForCausalLM",
|
||||
"MistralLarge3ForCausalLM",
|
||||
"PixtralForConditionalGeneration",
|
||||
]:
|
||||
if is_deepseek_nsa(hf_config):
|
||||
if (
|
||||
self.attention_backend is None
|
||||
@@ -1050,7 +1061,7 @@ class ServerArgs:
|
||||
# Default DeepSeek V3/R1 native FP8 when not explicitly set,
|
||||
# Because we need this condition for an assertion in
|
||||
# flashinfer_trtllm MoE runner backend.
|
||||
if quant_method is None:
|
||||
if quant_method is None and model_arch == "DeepseekV3ForCausalLM":
|
||||
self.quantization = "fp8"
|
||||
logger.info(
|
||||
"Quantization not specified, default to fp8 for DeepSeek on sm100"
|
||||
@@ -1693,6 +1704,8 @@ class ServerArgs:
|
||||
"Glm4MoeForCausalLM",
|
||||
"BailingMoeForCausalLM",
|
||||
"BailingMoeV2ForCausalLM",
|
||||
"MistralLarge3ForCausalLM",
|
||||
"PixtralForConditionalGeneration",
|
||||
]:
|
||||
if self.speculative_draft_model_path is None:
|
||||
self.speculative_draft_model_path = self.model_path
|
||||
@@ -1933,6 +1946,8 @@ class ServerArgs:
|
||||
"DeepseekV2ForCausalLM",
|
||||
"DeepseekV3ForCausalLM",
|
||||
"DeepseekV32ForCausalLM",
|
||||
"MistralLarge3ForCausalLM",
|
||||
"PixtralForConditionalGeneration",
|
||||
]
|
||||
except Exception:
|
||||
pass
|
||||
@@ -4526,6 +4541,8 @@ def auto_choose_speculative_params(self: ServerArgs):
|
||||
"GptOssForCausalLM",
|
||||
"BailingMoeForCausalLM",
|
||||
"BailingMoeV2ForCausalLM",
|
||||
"MistralLarge3ForCausalLM",
|
||||
"PixtralForConditionalGeneration",
|
||||
]:
|
||||
# The default value for deepseek and gpt-oss
|
||||
return (3, 1, 4)
|
||||
|
||||
Reference in New Issue
Block a user