model(vlm): mistral 3.1 (#5099)
Co-authored-by: KivenChen <sleigh-queue-0y@icloud.com>
This commit is contained in:
@@ -13,6 +13,12 @@
|
||||
# ==============================================================================
|
||||
"""Inference-only Mistral model."""
|
||||
|
||||
from typing import List, Union
|
||||
|
||||
import torch
|
||||
from transformers.models.mistral3.modeling_mistral3 import Mistral3MultiModalProjector
|
||||
|
||||
from sglang.srt.managers.schedule_batch import MultimodalDataItem
|
||||
from sglang.srt.models.llama import LlamaForCausalLM
|
||||
|
||||
|
||||
@@ -20,4 +26,68 @@ class MistralForCausalLM(LlamaForCausalLM):
|
||||
pass
|
||||
|
||||
|
||||
EntryClass = MistralForCausalLM
|
||||
class Mistral3ForConditionalGeneration:
|
||||
MULTIMODAL_PROJECTOR_TYPE = Mistral3MultiModalProjector
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
# lazy load inner class
|
||||
# to bypass circular import
|
||||
from sglang.srt.models.llava import LlavaForConditionalGeneration
|
||||
|
||||
# override config: mistral's projector adds patchmerger that doesn't require padding
|
||||
kwargs["config"].vision_config.pad_image_border = False
|
||||
|
||||
self.inner = LlavaForConditionalGeneration(**kwargs)
|
||||
self.inner.multi_modal_projector = self.MULTIMODAL_PROJECTOR_TYPE(
|
||||
kwargs["config"]
|
||||
)
|
||||
self.inner.get_image_feature = self.get_image_feature
|
||||
|
||||
def get_image_feature(self, items: List[MultimodalDataItem]) -> torch.Tensor:
|
||||
"""Extract features from image inputs.
|
||||
|
||||
Args:
|
||||
items: List of MultimodalDataItem objects containing image data
|
||||
Note that an item can be either "image" or "multi-images"
|
||||
|
||||
Returns:
|
||||
torch.Tensor: features from image inputs, concatenated
|
||||
"""
|
||||
features = []
|
||||
for item in items:
|
||||
# in each item, we assume pixel_values is always batched
|
||||
pixel_values, image_sizes = item.pixel_values, item.image_sizes
|
||||
image_outputs = self.vision_tower(
|
||||
pixel_values, image_sizes, output_hidden_states=True
|
||||
)
|
||||
selected_image_feature = image_outputs.hidden_states[
|
||||
self.vision_feature_layer
|
||||
]
|
||||
|
||||
if self.vision_feature_select_strategy in ["default", "patch"]:
|
||||
selected_image_feature = selected_image_feature[:, 1:]
|
||||
elif self.vision_feature_select_strategy == "full":
|
||||
selected_image_feature = selected_image_feature
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Unexpected select feature: {self.vision_feature_select_strategy}"
|
||||
)
|
||||
features.append(
|
||||
self.multi_modal_projector(
|
||||
selected_image_feature.squeeze(0), image_sizes
|
||||
)
|
||||
)
|
||||
ret = torch.cat(features, dim=0)
|
||||
return ret
|
||||
|
||||
def __getattr__(self, name):
|
||||
return getattr(self.inner, name)
|
||||
|
||||
def __hasattr__(self, name):
|
||||
return hasattr(self.inner, name)
|
||||
|
||||
def __call__(self, *args, **kwargs):
|
||||
return self.inner(*args, **kwargs)
|
||||
|
||||
|
||||
EntryClass = [MistralForCausalLM, Mistral3ForConditionalGeneration]
|
||||
|
||||
Reference in New Issue
Block a user