From 6d37e70883375c83fcb33b3eb0ed3838d69ea229 Mon Sep 17 00:00:00 2001 From: Xinyuan Tong <115166877+JustinTong0323@users.noreply.github.com> Date: Thu, 4 Dec 2025 22:31:26 +0000 Subject: [PATCH] ministral3 (#14251) Signed-off-by: Xinyuan Tong Co-authored-by: Yueming Yuan --- benchmark/mmmu/bench_sglang.py | 16 +- benchmark/mmmu/eval_utils.py | 25 ++- python/sglang/srt/configs/olmo3.py | 2 - python/sglang/srt/configs/qwen3_next.py | 2 - python/sglang/srt/layers/quantization/fp8.py | 7 +- .../sglang/srt/layers/quantization/utils.py | 6 +- .../sglang/srt/model_executor/model_runner.py | 6 +- .../sglang/srt/model_loader/weight_utils.py | 8 + python/sglang/srt/models/llama.py | 5 + python/sglang/srt/models/ministral3.py | 157 ++++++++++++++++++ python/sglang/srt/models/pixtral.py | 2 + .../srt/multimodal/processors/pixtral.py | 3 +- test/srt/models/test_ministral3_models.py | 31 ++++ test/srt/run_suite.py | 1 + 14 files changed, 245 insertions(+), 26 deletions(-) create mode 100644 python/sglang/srt/models/ministral3.py create mode 100644 test/srt/models/test_ministral3_models.py diff --git a/benchmark/mmmu/bench_sglang.py b/benchmark/mmmu/bench_sglang.py index 9a0bf4529..d9426ae5a 100644 --- a/benchmark/mmmu/bench_sglang.py +++ b/benchmark/mmmu/bench_sglang.py @@ -83,9 +83,9 @@ async def process_sample( assert image is not None image_path = sample["image_path"] extra_body = None if lora_path is None else {"lora_path": lora_path} - response = await client.chat.completions.create( - model="default", - messages=[ + payload = { + "model": "default", + "messages": [ { "role": "user", "content": [ @@ -95,11 +95,11 @@ async def process_sample( ], } ], - temperature=0, - max_completion_tokens=sampling_params["max_new_tokens"], - max_tokens=sampling_params["max_new_tokens"], - extra_body=extra_body, - ) + "extra_body": extra_body, + } + if sampling_params: + payload.update(sampling_params) + response = await client.chat.completions.create(**payload) return sample, response.choices[0].message.content diff --git a/benchmark/mmmu/eval_utils.py b/benchmark/mmmu/eval_utils.py index 955a3bfa5..b3edd69fc 100644 --- a/benchmark/mmmu/eval_utils.py +++ b/benchmark/mmmu/eval_utils.py @@ -36,7 +36,8 @@ class EvalArgs: profile: bool = False profile_number: int = 5 concurrency: int = 1 - max_new_tokens: int = 30 + max_new_tokens: Optional[int] = None + temperature: Optional[float] = None response_answer_regex: str = "(.*)" lora_path: Optional[str] = None @@ -101,6 +102,12 @@ class EvalArgs: default=EvalArgs.max_new_tokens, help="Maximum number of new tokens to generate per sample.", ) + parser.add_argument( + "--temperature", + type=float, + default=EvalArgs.temperature, + help="Sampling temperature for generation.", + ) parser.add_argument( "--response-answer-regex", type=str, @@ -241,19 +248,21 @@ def prepare_samples(eval_args: EvalArgs): def get_sampling_params(eval_args): - max_new_tokens = eval_args.max_new_tokens - temperature = 0.001 - extra_request_body = {} if eval_args.extra_request_body: extra_request_body = json.loads(eval_args.extra_request_body) - - return { - "temperature": temperature, - "max_new_tokens": max_new_tokens, + sampling_params = { **extra_request_body, } + if eval_args.max_new_tokens is not None and eval_args.max_new_tokens > 0: + sampling_params.update({"max_completion_tokens": eval_args.max_new_tokens}) + + if eval_args.temperature is not None: + sampling_params.update({"temperature": eval_args.temperature}) + + return sampling_params + # ----------- Process Multi-choice ------------- def parse_multi_choice_response(response, all_choices, index2ans): diff --git a/python/sglang/srt/configs/olmo3.py b/python/sglang/srt/configs/olmo3.py index 95e7c2537..864034219 100644 --- a/python/sglang/srt/configs/olmo3.py +++ b/python/sglang/srt/configs/olmo3.py @@ -17,7 +17,6 @@ import enum from transformers.configuration_utils import PretrainedConfig -from transformers.modeling_rope_utils import rope_config_validation from transformers.utils import logging logger = logging.get_logger(__name__) @@ -90,7 +89,6 @@ class Olmo3Config(PretrainedConfig): self.use_cache = use_cache self.rope_theta = rope_theta self.rope_scaling = rope_scaling - rope_config_validation(self) self.attention_bias = attention_bias self.attention_dropout = attention_dropout diff --git a/python/sglang/srt/configs/qwen3_next.py b/python/sglang/srt/configs/qwen3_next.py index cd1b6f1ea..8d0981c39 100644 --- a/python/sglang/srt/configs/qwen3_next.py +++ b/python/sglang/srt/configs/qwen3_next.py @@ -17,7 +17,6 @@ import enum from transformers.configuration_utils import PretrainedConfig -from transformers.modeling_rope_utils import rope_config_validation from transformers.utils import logging from sglang.srt.configs.mamba_utils import Mamba2CacheParams, Mamba2StateShape @@ -226,7 +225,6 @@ class Qwen3NextConfig(PretrainedConfig): self.attention_bias = attention_bias self.attention_dropout = attention_dropout self.head_dim = head_dim - rope_config_validation(self) # linear attention (gdn now part) self.linear_conv_kernel_dim = linear_conv_kernel_dim diff --git a/python/sglang/srt/layers/quantization/fp8.py b/python/sglang/srt/layers/quantization/fp8.py index d1a42236f..3338d5a41 100644 --- a/python/sglang/srt/layers/quantization/fp8.py +++ b/python/sglang/srt/layers/quantization/fp8.py @@ -156,7 +156,12 @@ class Fp8Config(QuantizationConfig): quant_method = cls.get_from_keys(config, ["quant_method"]) is_checkpoint_fp8_serialized = "fp8" in quant_method activation_scheme = cls.get_from_keys(config, ["activation_scheme"]) - ignored_layers = cls.get_from_keys_or(config, ["ignored_layers"], None) + ignored_layers = cls.get_from_keys_or( + config, ["ignored_layers", "modules_to_not_convert"], None + ) + if ignored_layers: + # hacking ministral + ignored_layers = [layer.replace("model.", "") for layer in ignored_layers] weight_block_size = cls.get_from_keys_or(config, ["weight_block_size"], None) return cls( is_checkpoint_fp8_serialized=is_checkpoint_fp8_serialized, diff --git a/python/sglang/srt/layers/quantization/utils.py b/python/sglang/srt/layers/quantization/utils.py index d407b95f2..fc81f3140 100644 --- a/python/sglang/srt/layers/quantization/utils.py +++ b/python/sglang/srt/layers/quantization/utils.py @@ -64,7 +64,9 @@ def is_layer_skipped( is_skipped = None for shard_prefix in shard_prefixes: - is_shard_skipped = shard_prefix in ignored_layers + is_shard_skipped = any( + ignored in shard_prefix for ignored in ignored_layers + ) if is_skipped is None: is_skipped = is_shard_skipped @@ -75,7 +77,7 @@ def is_layer_skipped( "to have the same precision." ) else: - is_skipped = prefix in ignored_layers + is_skipped = any(ignored in prefix for ignored in ignored_layers) if "gate_up_proj" in prefix: prefix_gate = prefix.replace("gate_up_proj", "gate_proj") prefix_up = prefix.replace("gate_up_proj", "up_proj") diff --git a/python/sglang/srt/model_executor/model_runner.py b/python/sglang/srt/model_executor/model_runner.py index 06f384e0e..568e9b848 100644 --- a/python/sglang/srt/model_executor/model_runner.py +++ b/python/sglang/srt/model_executor/model_runner.py @@ -583,8 +583,10 @@ class ModelRunner: quantization_config := getattr( self.model_config.hf_config, "quantization_config", None ) - ) is not None and "weight_block_size" in quantization_config: - weight_block_size_n = quantization_config["weight_block_size"][0] + ) is not None and ( + weight_block_size := quantization_config.get("weight_block_size", None) + ) is not None: + weight_block_size_n = weight_block_size[0] if self.tp_size % self.moe_ep_size != 0: raise ValueError( diff --git a/python/sglang/srt/model_loader/weight_utils.py b/python/sglang/srt/model_loader/weight_utils.py index ab4d42fda..4b02500e8 100644 --- a/python/sglang/srt/model_loader/weight_utils.py +++ b/python/sglang/srt/model_loader/weight_utils.py @@ -529,6 +529,14 @@ def filter_duplicate_safetensors_files( # torch state_dict to safetensors file holding that weight. index_file_name = os.path.join(hf_folder, index_file) if not os.path.isfile(index_file_name): + # NOTE: this is a trick of handling mistral model + # skip the unsupported consolidated.safetensors file + if len(hf_weights_files) == 2: + hf_weights_files.sort() + if hf_weights_files[0].endswith( + "consolidated.safetensors" + ) and hf_weights_files[1].endswith("model.safetensors"): + return [hf_weights_files[1]] return hf_weights_files # Iterate through the weight_map (weight_name: safetensors files) diff --git a/python/sglang/srt/models/llama.py b/python/sglang/srt/models/llama.py index dbf6968ee..4176af28d 100644 --- a/python/sglang/srt/models/llama.py +++ b/python/sglang/srt/models/llama.py @@ -570,6 +570,11 @@ class LlamaForCausalLM(nn.Module): params_dict = dict(self.named_parameters()) for name, loaded_weight in weights: + if name.endswith(".activation_scale"): + name = name.replace(".activation_scale", ".input_scale") + if name.endswith(".weight_scale_inv"): + name = name.replace(".weight_scale_inv", ".weight_scale") + layer_id = get_layer_id(name) if ( layer_id is not None diff --git a/python/sglang/srt/models/ministral3.py b/python/sglang/srt/models/ministral3.py new file mode 100644 index 000000000..460c7b30f --- /dev/null +++ b/python/sglang/srt/models/ministral3.py @@ -0,0 +1,157 @@ +from typing import Any, Dict, Optional + +import torch +from transformers import PretrainedConfig + +from sglang.srt.layers.quantization.base_config import QuantizationConfig +from sglang.srt.model_executor.forward_batch_info import ForwardBatch +from sglang.srt.models.llama import ( + LlamaAttention, + LlamaDecoderLayer, + LlamaForCausalLM, + LlamaModel, +) +from sglang.srt.utils import add_prefix, make_layers + + +def _get_llama_4_attn_scale( + positions_ids: torch.Tensor, beta: float, max_position_embeddings: int +) -> torch.Tensor: + scaling = 1 + beta * torch.log( + 1 + torch.floor(positions_ids / max_position_embeddings) + ) + return scaling.unsqueeze(-1) + + +class Ministral3Attention(LlamaAttention): + def __init__( + self, + config: PretrainedConfig, + hidden_size: int, + num_heads: int, + num_kv_heads: int, + layer_id: int = 0, + rope_theta: float = 1000000.0, + rope_scaling: Optional[Dict[str, Any]] = {}, + rope_is_neox_style: bool = True, + max_position_embeddings: int = 8192, + quant_config: Optional[QuantizationConfig] = None, + prefix: str = "", + bias: bool = False, + ) -> None: + super().__init__( + config, + hidden_size, + num_heads, + num_kv_heads, + layer_id, + rope_theta, + rope_scaling, + rope_is_neox_style, + max_position_embeddings, + quant_config, + prefix, + bias, + ) + # Ministral3 specific: llama 4 style scaling beta + self.llama_4_scaling_beta = None + if hasattr(config, "rope_parameters") and config.rope_parameters: + self.llama_4_scaling_beta = config.rope_parameters.get( + "llama_4_scaling_beta" + ) + + # sliding window + self.sliding_window = getattr(config, "sliding_window", None) + if self.sliding_window is not None: + # Update RadixAttention with sliding window if needed + # currently RadixAttention in sglang handles this mostly via logic in forward/flashinfer + pass + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + forward_batch: ForwardBatch, + ) -> torch.Tensor: + qkv, _ = self.qkv_proj(hidden_states) + q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1) + + # Apply RoPE + q, k = self.rotary_emb(positions, q, k) + + # Ministral3 / Llama 4 scaling + if self.llama_4_scaling_beta is not None: + scale = _get_llama_4_attn_scale( + positions, self.llama_4_scaling_beta, self.max_position_embeddings + ).to(q.dtype) + # q shape is [batch_size * seq_len, num_heads * head_dim] or [batch_size * seq_len, num_heads, head_dim] + # positions is [batch_size * seq_len] + # scale is [batch_size * seq_len, 1] + # We need to reshape q to apply scale correctly if it's flattened + # Assuming q is (total_tokens, num_heads * head_dim) + q = q.view(-1, self.num_heads, self.head_dim) + q = q * scale.unsqueeze(1) # Broadcast over heads + q = q.view(-1, self.num_heads * self.head_dim) + + attn_output = self.attn(q, k, v, forward_batch) + output, _ = self.o_proj(attn_output) + return output + + +class Ministral3DecoderLayer(LlamaDecoderLayer): + def __init__(self, config, layer_id=0, quant_config=None, prefix=""): + super().__init__(config, layer_id, quant_config, prefix) + self.self_attn = Ministral3Attention( + config=config, + hidden_size=self.hidden_size, + num_heads=config.num_attention_heads, + num_kv_heads=config.num_key_value_heads, + layer_id=layer_id, + rope_theta=getattr(config, "rope_parameters", {}).get( + "rope_theta", 1000000.0 + ), + rope_scaling=getattr( + config, "rope_parameters", {} + ), # rope_scaling is rope_parameters in Ministral3Config + max_position_embeddings=getattr( + config, "original_max_position_embeddings", 16384 + ), + quant_config=quant_config, + prefix=add_prefix("self_attn", prefix), + bias=getattr(config, "attention_bias", False) + or getattr(config, "bias", False), + ) + + +class Ministral3Model(LlamaModel): + def __init__( + self, + config: PretrainedConfig, + quant_config: Optional[QuantizationConfig] = None, + prefix: str = "", + ) -> None: + # Override layer creation to use Ministral3Attention + super().__init__(config, quant_config, prefix) + + self.layers, self.start_layer, self.end_layer = make_layers( + config.num_hidden_layers, + lambda idx, prefix: Ministral3DecoderLayer( + config=config, quant_config=quant_config, layer_id=idx, prefix=prefix + ), + pp_rank=self.pp_group.rank_in_group, + pp_size=self.pp_group.world_size, + prefix="model.layers", + ) + + +class Ministral3ForCausalLM(LlamaForCausalLM): + def _init_model( + self, + config: PretrainedConfig, + quant_config: Optional[QuantizationConfig] = None, + prefix: str = "", + ): + return Ministral3Model(config, quant_config, prefix=prefix) + + +EntryClass = [Ministral3ForCausalLM] diff --git a/python/sglang/srt/models/pixtral.py b/python/sglang/srt/models/pixtral.py index 1f9c236bb..c59770f45 100644 --- a/python/sglang/srt/models/pixtral.py +++ b/python/sglang/srt/models/pixtral.py @@ -514,6 +514,8 @@ class PixtralHFTransformerBlock(nn.Module): dropout=0.0, use_context_forward=False, flatten_batch=False, + qkv_bias=False, + proj_bias=False, prefix=f"{prefix}.attention", ) diff --git a/python/sglang/srt/multimodal/processors/pixtral.py b/python/sglang/srt/multimodal/processors/pixtral.py index 6b6ab34ad..b923ff342 100644 --- a/python/sglang/srt/multimodal/processors/pixtral.py +++ b/python/sglang/srt/multimodal/processors/pixtral.py @@ -56,7 +56,8 @@ class PixtralProcessor(BaseMultimodalProcessor): self.patch_size = self.vision_config.patch_size self._processor.patch_size = self.patch_size - self._processor.spatial_merge_size = self.vision_config.spatial_merge_size + if hasattr(self.vision_config, "spatial_merge_size"): + self._processor.spatial_merge_size = self.vision_config.spatial_merge_size self.mm_tokens = MultimodalSpecialTokens( image_token=_processor.image_token, diff --git a/test/srt/models/test_ministral3_models.py b/test/srt/models/test_ministral3_models.py new file mode 100644 index 000000000..6f6ca51b4 --- /dev/null +++ b/test/srt/models/test_ministral3_models.py @@ -0,0 +1,31 @@ +import unittest +from types import SimpleNamespace + +from sglang.test.gsm8k_mixin import GSM8KMixin +from sglang.test.mmmu_vlm_mixin import MMMUVLMMixin +from sglang.test.test_utils import CustomTestCase + +MODEL = "mistralai/Ministral-3-3B-Instruct-2512" + + +class TestMinistral3TextOnly(GSM8KMixin, CustomTestCase): + accuracy = 0.6 + model = MODEL + other_args = ["--trust-remote-code"] + + +class TestMinistral3MMMU(MMMUVLMMixin, CustomTestCase): + accuracy = 0.3 + model = MODEL + other_args = ["--trust-remote-code"] + mmmu_args = ["--limit=0.1"] + """`--limit=0.1`: 10 percent of each task - this is fine for testing since the nominal result isn't interesting - this run is just to prevent relative regressions.""" + + def test_vlm_mmmu_benchmark(self): + self._run_vlm_mmmu_test( + SimpleNamespace(model=self.model, mmmu_accuracy=self.accuracy), "./logs" + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/run_suite.py b/test/srt/run_suite.py index 0e45ff663..5c899a6c2 100644 --- a/test/srt/run_suite.py +++ b/test/srt/run_suite.py @@ -214,6 +214,7 @@ suites = { TestFile("test_moe_eval_accuracy_large.py"), TestFile("test_vision_openai_server_common.py"), TestFile("test_profile_v2.py"), + TestFile("models/test_ministral3_models.py"), ], }