chenchenghao/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[model-gateway] add mistral 3 image processor (#14445)

Browse Source 
Co-authored-by: Chang Su <chang.s.su@oracle.com>
This commit is contained in:
Simo Lin
2025-12-04 13:16:04 -08:00
committed by GitHub
parent eb85fa6daf
commit 29c6c2ea9d
10 changed files with 825 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

64
sgl-router/scripts/generate_vision_golden.py
View File

@@ -66,6 +66,11 @@ MODELS = {
"processor_class": "Llama4ImageProcessorFast",
"description": "Tile-based processing with 336x336 tiles and global tile",
},
"pixtral": {
"model_id": "mistralai/Pixtral-12B-2409",
"processor_class": "PixtralImageProcessor",
"description": "Dynamic resolution with CLIP normalization and bicubic resize",
},
}
# Default test images
@@ -547,6 +552,64 @@ def generate_golden_llama4_vision(image_path: str, output_dir: str) -> dict:
return result
def generate_golden_pixtral(image_path: str, output_dir: str) -> dict:
"""Generate golden output for Pixtral/Mistral3 Vision.
Pixtral uses dynamic resolution processing:
1. If image exceeds longest_edge (default 1024), scale down proportionally
2. Resize to dimensions that are multiples of patch_size (default 16)
3. Use bicubic interpolation for resize
4. Normalize with CLIP mean/std
Output:
- pixel_values: [1, 3, H, W] where H, W are multiples of patch_size
- image_sizes: [(H, W)]
Token count: (H / patch_size) * (W / patch_size)
"""
from transformers import PixtralImageProcessor
processor = PixtralImageProcessor.from_pretrained("mistral-community/pixtral-12b")
image = Image.open(image_path).convert("RGB")
original_size = image.size
# Process image
outputs = processor(images=image, return_tensors="np")
pixel_values = outputs["pixel_values"]
image_sizes = outputs.get("image_sizes")
result = {
"pixel_values": pixel_values,
"original_size": original_size,
"processor_config": processor.to_dict(),
}
if image_sizes is not None:
result["image_sizes"] = np.array(image_sizes)
# Calculate num_tokens from image_sizes
if image_sizes is not None:
h, w = image_sizes[0]
patch_size = getattr(processor, "patch_size", {"height": 16, "width": 16})
if isinstance(patch_size, dict):
patch_h = patch_size.get("height", 16)
patch_w = patch_size.get("width", 16)
else:
patch_h = patch_w = patch_size
num_tokens = (h // patch_h) * (w // patch_w)
result["num_tokens"] = num_tokens
# Add debug info
result["config_info"] = {
"longest_edge": processor.size.get("longest_edge", 1024),
"patch_size": processor.patch_size,
"image_mean": processor.image_mean,
"image_std": processor.image_std,
}
return result
def generate_for_model(model_key: str, image_paths: list, output_dir: str):
"""Generate golden outputs for a specific model."""
print(f"\nGenerating golden outputs for {model_key}...")
@@ -560,6 +623,7 @@ def generate_for_model(model_key: str, image_paths: list, output_dir: str):
"phi3_vision": generate_golden_phi3_vision,
"phi4_vision": generate_golden_phi4_vision,
"llama4_vision": generate_golden_llama4_vision,
"pixtral": generate_golden_pixtral,
}.get(model_key)
if generator_fn is None:

2
sgl-router/src/multimodal/vision/mod.rs
View File

@@ -41,6 +41,6 @@ pub use image_processor::{
pub use preprocessor_config::PreProcessorConfig;
pub use processors::{
Llama4VisionProcessor, LlavaNextProcessor, LlavaProcessor, Phi3VisionProcessor,
Phi4VisionProcessor, Qwen2VLProcessor, Qwen3VLProcessor,
Phi4VisionProcessor, PixtralProcessor, Qwen2VLProcessor, Qwen3VLProcessor,
};
pub use transforms::TransformError;

107
sgl-router/src/multimodal/vision/preprocessor_config.rs
View File

@@ -6,10 +6,97 @@
use std::collections::HashMap;
use image::imageops::FilterType;
use serde::Deserialize;
use serde::{Deserialize, Deserializer};
use super::transforms;
/// Struct to represent patch_size as dict {"height": x, "width": y}
#[derive(Debug, Clone, Deserialize, Default)]
pub struct PatchSize {
pub height: Option<u32>,
pub width: Option<u32>,
}
/// Custom deserializer for patch_size that handles both integer and dict formats.
/// - Integer format: `"patch_size": 16` -> PatchSize { height: 16, width: 16 }
/// - Dict format: `"patch_size": {"height": 16, "width": 16}` -> PatchSize { height: 16, width: 16 }
fn deserialize_patch_size<'de, D>(deserializer: D) -> Result<Option<PatchSize>, D::Error>
where
D: Deserializer<'de>,
{
use std::fmt;
use serde::de::{self, MapAccess, Visitor};
struct PatchSizeVisitor;
impl<'de> Visitor<'de> for PatchSizeVisitor {
type Value = Option<PatchSize>;
fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
formatter.write_str("an integer, a dict with height/width, or null")
}
fn visit_none<E>(self) -> Result<Self::Value, E>
where
E: de::Error,
{
Ok(None)
}
fn visit_unit<E>(self) -> Result<Self::Value, E>
where
E: de::Error,
{
Ok(None)
}
fn visit_i64<E>(self, value: i64) -> Result<Self::Value, E>
where
E: de::Error,
{
let v = value as u32;
Ok(Some(PatchSize {
height: Some(v),
width: Some(v),
}))
}
fn visit_u64<E>(self, value: u64) -> Result<Self::Value, E>
where
E: de::Error,
{
let v = value as u32;
Ok(Some(PatchSize {
height: Some(v),
width: Some(v),
}))
}
fn visit_map<M>(self, mut map: M) -> Result<Self::Value, M::Error>
where
M: MapAccess<'de>,
{
let mut height = None;
let mut width = None;
while let Some(key) = map.next_key::<String>()? {
match key.as_str() {
"height" => height = Some(map.next_value::<u32>()?),
"width" => width = Some(map.next_value::<u32>()?),
_ => {
let _ = map.next_value::<de::IgnoredAny>()?;
}
}
}
Ok(Some(PatchSize { height, width }))
}
}
deserializer.deserialize_any(PatchSizeVisitor)
}
/// HuggingFace preprocessor_config.json structure.
///
/// This struct captures the common fields across different vision model processors.
@@ -73,8 +160,9 @@ pub struct PreProcessorConfig {
// Model-specific fields
// =====================
/// Vision encoder patch size (typically 14 or 16)
#[serde(default)]
pub patch_size: Option<usize>,
/// Can be an integer or a dict {"height": x, "width": y}
#[serde(default, deserialize_with = "deserialize_patch_size")]
pub patch_size: Option<PatchSize>,
/// Qwen-VL: merge size for token reduction
#[serde(default)]
@@ -151,6 +239,17 @@ impl PreProcessorConfig {
serde_json::from_value(value)
}
/// Get patch size as a simple usize.
///
/// Returns the height value from PatchSize if available, falling back to provided default.
pub fn get_patch_size(&self, default: usize) -> usize {
self.patch_size
.as_ref()
.and_then(|p| p.height)
.map(|h| h as usize)
.unwrap_or(default)
}
/// Get image mean as fixed array, with fallback to CLIP defaults.
pub fn get_image_mean(&self) -> [f64; 3] {
self.image_mean
@@ -310,7 +409,7 @@ mod tests {
assert_eq!(config.min_pixels, Some(200704));
assert_eq!(config.max_pixels, Some(1003520));
assert_eq!(config.patch_size, Some(14));
assert_eq!(config.get_patch_size(0), 14);
assert_eq!(config.merge_size, Some(2));
assert!((config.get_rescale_factor() - 1.0 / 255.0).abs() < 1e-10);
}

2
sgl-router/src/multimodal/vision/processors/llava.rs
View File

@@ -308,7 +308,7 @@ impl ImagePreProcessor for LlavaProcessor {
config: &PreProcessorConfig,
) -> usize {
// For LLaVA 1.5, token count is based on processed image size and patch size
let patch_size = config.patch_size.unwrap_or(self.patch_size as usize) as u32;
let patch_size = config.get_patch_size(self.patch_size as usize) as u32;
let image_size = config
.get_target_size()
.map(|(h, _w)| h)

3
sgl-router/src/multimodal/vision/processors/mod.rs
View File

@@ -13,11 +13,13 @@
//! - **Phi3-Vision** (`phi3_vision`): Dynamic HD transform with 336x336 tiles
//! - **Phi4-Vision** (`phi4_vision`): Dynamic HD transform with 448x448 tiles and SiGLIP encoder
//! - **LLaMA 4 Vision** (`llama4_vision`): Tile-based processing with 336x336 tiles and global tile
//! - **Pixtral/Mistral3** (`pixtral`): CLIP-based preprocessing with dynamic resolution
pub mod llama4_vision;
pub mod llava;
pub mod phi3_vision;
pub mod phi4_vision;
pub mod pixtral;
pub mod qwen2_vl;
pub mod qwen3_vl;
pub mod qwen_vl_base;
@@ -26,5 +28,6 @@ pub use llama4_vision::Llama4VisionProcessor;
pub use llava::{ImageAspectRatio, LlavaNextProcessor, LlavaProcessor};
pub use phi3_vision::Phi3VisionProcessor;
pub use phi4_vision::Phi4VisionProcessor;
pub use pixtral::PixtralProcessor;
pub use qwen2_vl::Qwen2VLProcessor;
pub use qwen3_vl::Qwen3VLProcessor;

411
sgl-router/src/multimodal/vision/processors/pixtral.rs Normal file
View File

@@ -0,0 +1,411 @@
//! Pixtral/Mistral3 Vision image processor implementation.
//!
//! This module implements the image preprocessing for Pixtral/Mistral3 models,
//! matching the behavior of HuggingFace's `PixtralImageProcessor`.
//!
//! Key characteristics:
//! - CLIP normalization: mean [0.48145466, 0.4578275, 0.40821073], std [0.26862954, 0.26130258, 0.27577711]
//! - Bicubic resampling for resize
//! - Images resized to fit within longest_edge (default 1024)
//! - Output dimensions are multiples of patch_size (default 16)
//! - No tiling - single image output per input
use std::collections::HashMap;
use image::{imageops::FilterType, DynamicImage};
use ndarray::{Array4, IxDyn};
use crate::multimodal::vision::{
image_processor::{ImagePreProcessor, ModelSpecificValue, PreprocessedImages},
preprocessor_config::PreProcessorConfig,
transforms::{self, TransformError},
};
/// Default normalization mean values (CLIP)
const DEFAULT_IMAGE_MEAN: [f64; 3] = [0.48145466, 0.4578275, 0.40821073];
/// Default normalization std values (CLIP)
const DEFAULT_IMAGE_STD: [f64; 3] = [0.26862954, 0.26130258, 0.27577711];
/// Default longest edge for resize
const DEFAULT_LONGEST_EDGE: u32 = 1024;
/// Default patch size
const DEFAULT_PATCH_SIZE: u32 = 16;
/// Pixtral/Mistral3 Vision image processor.
///
/// This processor handles image preprocessing for Pixtral and Mistral3 vision models.
/// Unlike tile-based processors (Phi3, LLaMA4), Pixtral processes images at their
/// natural resolution (up to a maximum), preserving aspect ratio.
#[derive(Debug, Clone)]
pub struct PixtralProcessor {
/// Maximum dimension for the longest edge
longest_edge: u32,
/// Patch size for calculating output dimensions
patch_size: u32,
/// Normalization mean values
image_mean: [f64; 3],
/// Normalization std values
image_std: [f64; 3],
}
impl Default for PixtralProcessor {
fn default() -> Self {
Self::new()
}
}
impl PixtralProcessor {
/// Creates a new Pixtral processor with default settings.
pub fn new() -> Self {
Self {
longest_edge: DEFAULT_LONGEST_EDGE,
patch_size: DEFAULT_PATCH_SIZE,
image_mean: DEFAULT_IMAGE_MEAN,
image_std: DEFAULT_IMAGE_STD,
}
}
/// Creates a processor from a HuggingFace preprocessor config.
pub fn from_preprocessor_config(config: &PreProcessorConfig) -> Self {
let longest_edge = config
.size
.as_ref()
.and_then(|s| s.get("longest_edge").copied())
.unwrap_or(DEFAULT_LONGEST_EDGE);
// Patch size uses the new PatchSize type from config
let patch_size = config.get_patch_size(DEFAULT_PATCH_SIZE as usize) as u32;
let image_mean = config
.image_mean
.as_ref()
.filter(|m| m.len() >= 3)
.map(|m| [m[0], m[1], m[2]])
.unwrap_or(DEFAULT_IMAGE_MEAN);
let image_std = config
.image_std
.as_ref()
.filter(|s| s.len() >= 3)
.map(|s| [s[0], s[1], s[2]])
.unwrap_or(DEFAULT_IMAGE_STD);
Self {
longest_edge,
patch_size,
image_mean,
image_std,
}
}
/// Calculates the target output size for an image.
///
/// The image is resized to fit within `longest_edge` while preserving aspect ratio.
/// The output dimensions are then adjusted to be multiples of `patch_size`.
fn get_resize_output_size(&self, height: u32, width: u32) -> (u32, u32) {
let max_size = self.longest_edge;
let patch_size = self.patch_size;
// Calculate ratio for scaling down (only if larger than max_size)
let ratio = f64::max(
height as f64 / max_size as f64,
width as f64 / max_size as f64,
);
let (new_height, new_width) = if ratio > 1.0 {
// Scale down using floor to ensure we don't exceed max_size
let new_height = (height as f64 / ratio).floor() as u32;
let new_width = (width as f64 / ratio).floor() as u32;
(new_height, new_width)
} else {
(height, width)
};
// Calculate number of patches in each dimension
// Using: num_tokens = (dim - 1) / patch_size + 1 (i.e., ceiling division)
let num_height_tokens = (new_height.max(1) - 1) / patch_size + 1;
let num_width_tokens = (new_width.max(1) - 1) / patch_size + 1;
// Final size is patches * patch_size
(
num_height_tokens * patch_size,
num_width_tokens * patch_size,
)
}
/// Processes a single image through the Pixtral pipeline.
fn process_single_image(
&self,
image: &DynamicImage,
) -> Result<(Array4<f32>, (usize, usize)), TransformError> {
let (orig_width, orig_height) = (image.width(), image.height());
// Step 1: Calculate output size
let (target_h, target_w) = self.get_resize_output_size(orig_height, orig_width);
// Step 2: Resize image using bicubic interpolation
let resized = image.resize_exact(target_w, target_h, FilterType::CatmullRom);
// Step 3: Convert to tensor (0-1 range) and normalize
let mut tensor = transforms::to_tensor(&resized);
transforms::normalize(&mut tensor, &self.image_mean, &self.image_std);
// Step 4: Reshape to (1, C, H, W)
let (c, h, w) = (tensor.shape()[0], tensor.shape()[1], tensor.shape()[2]);
let output = tensor
.into_shape_with_order((1, c, h, w))
.map_err(|e| TransformError::ShapeError(e.to_string()))?;
Ok((output, (target_h as usize, target_w as usize)))
}
}
impl ImagePreProcessor for PixtralProcessor {
fn default_mean(&self) -> [f64; 3] {
self.image_mean
}
fn default_std(&self) -> [f64; 3] {
self.image_std
}
fn preprocess(
&self,
images: &[DynamicImage],
config: &PreProcessorConfig,
) -> Result<PreprocessedImages, TransformError> {
if images.is_empty() {
return Err(TransformError::InvalidShape {
expected: "non-empty image batch".to_string(),
actual: vec![0],
});
}
// Apply config overrides if present
let processor = if config.size.is_some()
|| config.patch_size.is_some()
|| config.image_mean.is_some()
|| config.image_std.is_some()
{
Self::from_preprocessor_config(config)
} else {
self.clone()
};
let mut all_pixel_values = Vec::new();
let mut all_image_sizes = Vec::new();
let mut original_sizes = Vec::new();
let mut num_img_tokens = Vec::new();
for image in images {
let (pixels, size) = processor.process_single_image(image)?;
let tokens = processor.calculate_num_tokens(image.width(), image.height(), config);
all_pixel_values.push(pixels);
all_image_sizes.push(size);
original_sizes.push((image.height(), image.width()));
num_img_tokens.push(tokens);
}
// Pad images to the same size for batching
let max_height = all_image_sizes.iter().map(|(h, _)| *h).max().unwrap_or(0);
let max_width = all_image_sizes.iter().map(|(_, w)| *w).max().unwrap_or(0);
// Create batch tensor with padding
let batch_size = all_pixel_values.len();
let channels = 3;
let mut batch_tensor =
ndarray::ArrayD::<f32>::zeros(IxDyn(&[batch_size, channels, max_height, max_width]));
for (i, (pixels, (h, w))) in all_pixel_values
.iter()
.zip(all_image_sizes.iter())
.enumerate()
{
// Copy the image data into the batch (top-left aligned, zero-padded)
for c in 0..channels {
for y in 0..*h {
for x in 0..*w {
batch_tensor[[i, c, y, x]] = pixels[[0, c, y, x]];
}
}
}
}
// Store image sizes as model-specific data
let mut model_specific = HashMap::new();
let image_sizes_flat: Vec<i64> = all_image_sizes
.iter()
.flat_map(|&(h, w)| vec![h as i64, w as i64])
.collect();
model_specific.insert(
"image_sizes".to_string(),
ModelSpecificValue::IntTensor {
data: image_sizes_flat,
shape: vec![batch_size, 2],
},
);
Ok(PreprocessedImages {
pixel_values: batch_tensor,
num_img_tokens,
image_sizes: original_sizes,
model_specific,
})
}
fn calculate_num_tokens(&self, width: u32, height: u32, config: &PreProcessorConfig) -> usize {
let processor = Self::from_preprocessor_config(config);
let (target_h, target_w) = processor.get_resize_output_size(height, width);
let patch_size = processor.patch_size;
// Number of tokens = num_patches_h * num_patches_w
let num_patches_h = target_h / patch_size;
let num_patches_w = target_w / patch_size;
(num_patches_h * num_patches_w) as usize
}
fn model_name(&self) -> &'static str {
"pixtral"
}
fn get_processed_size(&self, _config: &PreProcessorConfig) -> Option<(u32, u32)> {
// Pixtral has dynamic size based on input
None
}
}
#[cfg(test)]
mod tests {
use image::{Rgb, RgbImage};
use super::*;
fn create_test_image(width: u32, height: u32) -> DynamicImage {
let mut img = RgbImage::new(width, height);
for y in 0..height {
for x in 0..width {
let r = ((x * 255) / width.max(1)) as u8;
let g = ((y * 255) / height.max(1)) as u8;
let b = (((x + y) * 128) / (width + height).max(1)) as u8;
img.put_pixel(x, y, Rgb([r, g, b]));
}
}
DynamicImage::ImageRgb8(img)
}
#[test]
fn test_resize_output_size_small_image() {
let processor = PixtralProcessor::new();
// Small image that doesn't need resizing - just pad to patch boundary
// 100x100 -> patches: ceil(100/16) = 7, output: 7*16 = 112
let (h, w) = processor.get_resize_output_size(100, 100);
assert_eq!((h, w), (112, 112));
}
#[test]
fn test_resize_output_size_large_image() {
let processor = PixtralProcessor::new();
// Large image that needs resizing
// 2048x1024: ratio = 2048/1024 = 2.0
// scaled: 2048/2 = 1024, 1024/2 = 512
// patches h: ceil(1024/16) = 64, patches w: ceil(512/16) = 32
// output: 64*16 = 1024, 32*16 = 512
let (h, w) = processor.get_resize_output_size(2048, 1024);
assert_eq!((h, w), (1024, 512));
}
#[test]
fn test_resize_output_size_at_limit() {
let processor = PixtralProcessor::new();
// Image exactly at limit
// 1024x768: ratio = max(1024/1024, 768/1024) = 1.0
// No resize needed
// patches h: ceil(1024/16) = 64, patches w: ceil(768/16) = 48
// output: 64*16 = 1024, 48*16 = 768
let (h, w) = processor.get_resize_output_size(1024, 768);
assert_eq!((h, w), (1024, 768));
}
#[test]
fn test_process_single_image() {
let processor = PixtralProcessor::new();
let image = create_test_image(200, 150);
let (tensor, size) = processor.process_single_image(&image).unwrap();
// 200x150 -> patches h: ceil(150/16) = 10, patches w: ceil(200/16) = 13
// output: 10*16 = 160, 13*16 = 208
assert_eq!(size, (160, 208));
assert_eq!(tensor.shape(), &[1, 3, 160, 208]);
}
#[test]
fn test_preprocess_batch() {
let processor = PixtralProcessor::new();
let config = PreProcessorConfig::default();
let images = vec![create_test_image(200, 150), create_test_image(300, 100)];
let result = processor.preprocess(&images, &config).unwrap();
// First image: 150x200 -> 160x208
// Second image: 100x300 -> 112x304 (ceil(100/16)=7, ceil(300/16)=19)
// Batch padded to max: 160x304
assert_eq!(result.pixel_values.shape()[0], 2); // batch size
assert_eq!(result.pixel_values.shape()[1], 3); // channels
}
#[test]
fn test_normalization_values() {
let processor = PixtralProcessor::new();
// Verify CLIP normalization values
assert!((processor.image_mean[0] - 0.48145466).abs() < 1e-6);
assert!((processor.image_mean[1] - 0.4578275).abs() < 1e-6);
assert!((processor.image_mean[2] - 0.40821073).abs() < 1e-6);
assert!((processor.image_std[0] - 0.26862954).abs() < 1e-6);
assert!((processor.image_std[1] - 0.26130258).abs() < 1e-6);
assert!((processor.image_std[2] - 0.27577711).abs() < 1e-6);
}
#[test]
fn test_from_config() {
let mut size = HashMap::new();
size.insert("longest_edge".to_string(), 2048u32);
let config = PreProcessorConfig {
size: Some(size),
patch_size: Some(crate::multimodal::vision::preprocessor_config::PatchSize {
height: Some(14),
width: Some(14),
}),
image_mean: Some(vec![0.5, 0.5, 0.5]),
image_std: Some(vec![0.5, 0.5, 0.5]),
..Default::default()
};
let processor = PixtralProcessor::from_preprocessor_config(&config);
assert_eq!(processor.longest_edge, 2048);
assert_eq!(processor.patch_size, 14);
assert_eq!(processor.image_mean, [0.5, 0.5, 0.5]);
assert_eq!(processor.image_std, [0.5, 0.5, 0.5]);
}
#[test]
fn test_calculate_num_tokens() {
let processor = PixtralProcessor::new();
let config = PreProcessorConfig::default();
// 200x150 -> 208x160 -> 13*10 = 130 patches
let tokens = processor.calculate_num_tokens(200, 150, &config);
assert_eq!(tokens, 130);
}
}

16
sgl-router/src/multimodal/vision/processors/qwen2_vl.rs
View File

@@ -119,7 +119,7 @@ impl Qwen2VLProcessor {
pub fn from_preprocessor_config(config: &PreProcessorConfig) -> Self {
Self {
inner: QwenVLProcessorBase::new(QwenVLConfig {
patch_size: config.patch_size.unwrap_or(DEFAULT_PATCH_SIZE),
patch_size: config.get_patch_size(DEFAULT_PATCH_SIZE),
merge_size: config.merge_size.unwrap_or(DEFAULT_MERGE_SIZE),
min_pixels: config.min_pixels.unwrap_or(DEFAULT_MIN_PIXELS),
max_pixels: config.max_pixels.unwrap_or(DEFAULT_MAX_PIXELS),
@@ -245,7 +245,9 @@ mod tests {
use image::{Rgb, RgbImage};
use super::*;
use crate::multimodal::vision::image_processor::ModelSpecificValue;
use crate::multimodal::vision::{
image_processor::ModelSpecificValue, preprocessor_config::PatchSize,
};
fn create_test_image(width: u32, height: u32, color: Rgb<u8>) -> DynamicImage {
DynamicImage::from(RgbImage::from_pixel(width, height, color))
@@ -363,7 +365,10 @@ mod tests {
do_normalize: Some(true),
image_mean: Some(CLIP_MEAN.to_vec()),
image_std: Some(CLIP_STD.to_vec()),
patch_size: Some(14),
patch_size: Some(PatchSize {
height: Some(14),
width: Some(14),
}),
merge_size: Some(2),
min_pixels: Some(DEFAULT_MIN_PIXELS),
max_pixels: Some(DEFAULT_MAX_PIXELS),
@@ -418,7 +423,10 @@ mod tests {
#[test]
fn test_qwen2_vl_from_config() {
let config = PreProcessorConfig {
patch_size: Some(16),
patch_size: Some(PatchSize {
height: Some(16),
width: Some(16),
}),
merge_size: Some(4),
min_pixels: Some(100000),
max_pixels: Some(500000),

16
sgl-router/src/multimodal/vision/processors/qwen3_vl.rs
View File

@@ -120,7 +120,7 @@ impl Qwen3VLProcessor {
pub fn from_preprocessor_config(config: &PreProcessorConfig) -> Self {
Self {
inner: QwenVLProcessorBase::new(QwenVLConfig {
patch_size: config.patch_size.unwrap_or(DEFAULT_PATCH_SIZE),
patch_size: config.get_patch_size(DEFAULT_PATCH_SIZE),
merge_size: config.merge_size.unwrap_or(DEFAULT_MERGE_SIZE),
min_pixels: config.min_pixels.unwrap_or(DEFAULT_MIN_PIXELS),
max_pixels: config.max_pixels.unwrap_or(DEFAULT_MAX_PIXELS),
@@ -246,7 +246,9 @@ mod tests {
use image::{Rgb, RgbImage};
use super::*;
use crate::multimodal::vision::image_processor::ModelSpecificValue;
use crate::multimodal::vision::{
image_processor::ModelSpecificValue, preprocessor_config::PatchSize,
};
fn create_test_image(width: u32, height: u32, color: Rgb<u8>) -> DynamicImage {
DynamicImage::from(RgbImage::from_pixel(width, height, color))
@@ -338,7 +340,10 @@ mod tests {
do_normalize: Some(true),
image_mean: Some(QWEN3_MEAN.to_vec()),
image_std: Some(QWEN3_STD.to_vec()),
patch_size: Some(16),
patch_size: Some(PatchSize {
height: Some(16),
width: Some(16),
}),
merge_size: Some(2),
min_pixels: Some(DEFAULT_MIN_PIXELS),
max_pixels: Some(DEFAULT_MAX_PIXELS),
@@ -398,7 +403,10 @@ mod tests {
#[test]
fn test_qwen3_vl_from_config() {
let config = PreProcessorConfig {
patch_size: Some(16),
patch_size: Some(PatchSize {
height: Some(16),
width: Some(16),
}),
merge_size: Some(4),
min_pixels: Some(100000),
max_pixels: Some(500000),

3
sgl-router/src/multimodal/vision/transforms.rs
View File

@@ -24,6 +24,9 @@ pub enum TransformError {
#[error("Inconsistent tensor shapes in batch")]
InconsistentShapes,
#[error("Shape error: {0}")]
ShapeError(String),
}
pub type Result<T> = std::result::Result<T, TransformError>;

219
sgl-router/tests/vision_golden_tests.rs
View File

@@ -19,8 +19,8 @@ use std::{fs::File, io::Read, path::Path};
use ndarray::{Array4, Array5};
use sgl_model_gateway::multimodal::vision::{
image_processor::ModelSpecificValue, ImagePreProcessor, Llama4VisionProcessor, LlavaProcessor,
Phi3VisionProcessor, Phi4VisionProcessor, PreProcessorConfig, Qwen2VLProcessor,
Qwen3VLProcessor,
Phi3VisionProcessor, Phi4VisionProcessor, PixtralProcessor, PreProcessorConfig,
Qwen2VLProcessor, Qwen3VLProcessor,
};
/// Load a numpy .npz file and extract pixel_values
@@ -396,7 +396,7 @@ fn run_qwen2_vl_golden_test(image_name: &str) {
// Verify shapes match
let expected_num_patches = grid_t * grid_h * grid_w;
let patch_size = config.patch_size.unwrap_or(14);
let patch_size = config.get_patch_size(14);
let temporal_patch_size = config.temporal_patch_size.unwrap_or(2);
let expected_patch_features = 3 * temporal_patch_size * patch_size * patch_size;
@@ -580,7 +580,7 @@ fn run_qwen3_vl_golden_test(image_name: &str) {
// Verify shapes match (Qwen3-VL has patch_size=16)
let expected_num_patches = grid_t * grid_h * grid_w;
let patch_size = config.patch_size.unwrap_or(16);
let patch_size = config.get_patch_size(16);
let temporal_patch_size = config.temporal_patch_size.unwrap_or(2);
let expected_patch_features = 3 * temporal_patch_size * patch_size * patch_size;
@@ -1388,3 +1388,214 @@ fn test_llama4_vision_golden_odd_dims() {
fn test_llama4_vision_golden_grayscale() {
run_llama4_vision_golden_test("grayscale");
}
// ============================================================================
// Pixtral/Mistral3 Vision tests
// ============================================================================
/// Load image_sizes from npz file for Pixtral
fn load_pixtral_image_sizes(path: &Path) -> Vec<(usize, usize)> {
let file = File::open(path).expect("Failed to open golden file");
let mut npz = npyz::npz::NpzArchive::new(file).expect("Failed to parse npz");
let reader = npz
.by_name("image_sizes")
.expect("Failed to read npz")
.expect("No image_sizes");
let shape = reader.shape().to_vec();
// Read data as i64 vec (numpy default for int)
let data: Vec<i64> = reader.into_vec().expect("Failed to read array");
// Convert to Vec<(usize, usize)>
let num_images = shape[0] as usize;
(0..num_images)
.map(|i| (data[i * 2] as usize, data[i * 2 + 1] as usize))
.collect()
}
/// Run a Pixtral golden test for a specific image.
///
/// This test validates:
/// 1. Output shape matches (batch, 3, H, W)
/// 2. image_sizes match
/// 3. Pixel values match HuggingFace output
/// 4. Token count is correct
///
/// Pixtral processing:
/// - Longest edge: 1024 (default)
/// - Patch size: 16
/// - Normalization: CLIP mean/std
/// - No tiling - single output per image
fn run_pixtral_golden_test(image_name: &str) {
let golden_dir = Path::new("tests/fixtures/golden/pixtral");
let image_path = Path::new("tests/fixtures/images").join(format!("{}.jpg", image_name));
if !golden_dir.exists() || !image_path.exists() {
eprintln!(
"Golden test fixtures for pixtral/{} not found, skipping test",
image_name
);
eprintln!("Run: python scripts/generate_vision_golden.py --model pixtral");
return;
}
let npz_path = golden_dir.join(format!("golden_{}.npz", image_name));
let config = load_config(&golden_dir.join("preprocessor_config.json"));
// Load golden values
let golden_pixels = load_golden_npz(&npz_path);
let golden_shape: Vec<usize> = golden_pixels.shape().to_vec();
let golden_image_sizes = load_pixtral_image_sizes(&npz_path);
let golden_num_tokens = load_golden_num_tokens(&npz_path);
// Process image with our Rust processor
let image = image::open(&image_path).expect("Failed to open image");
let processor = PixtralProcessor::from_preprocessor_config(&config);
let result = processor
.preprocess(&[image], &config)
.expect("Processing failed");
// Check image_sizes from model_specific
let rust_image_sizes: Vec<(usize, usize)> = match result.model_specific.get("image_sizes") {
Some(ModelSpecificValue::IntTensor { data, shape }) => {
let num_images = shape[0];
(0..num_images)
.map(|i| (data[i * 2] as usize, data[i * 2 + 1] as usize))
.collect()
}
_ => panic!("Expected image_sizes in model_specific"),
};
println!(
"pixtral - {} image - Image sizes: golden={:?}, rust={:?}",
image_name, golden_image_sizes, rust_image_sizes
);
assert_eq!(
golden_image_sizes, rust_image_sizes,
"image_sizes mismatch for {}",
image_name
);
// Check num_tokens
let rust_num_tokens = result.num_img_tokens[0];
println!(
"pixtral - {} image - Tokens: golden={}, rust={}",
image_name, golden_num_tokens, rust_num_tokens
);
assert_eq!(
golden_num_tokens, rust_num_tokens,
"num_tokens mismatch for {}",
image_name
);
// Check output shape
let rust_shape = result.pixel_values.shape();
println!(
"pixtral - {} image - Shape: golden={:?}, rust={:?}",
image_name, golden_shape, rust_shape
);
// Pixtral outputs [batch, C, H, W] with padding to max size in batch
// Single image should match golden shape exactly
assert_eq!(rust_shape[0], 1, "Expected batch dim to be 1");
assert_eq!(rust_shape[1], golden_shape[1], "Channel mismatch");
assert!(
rust_shape[2] >= golden_shape[2],
"Height {} < golden height {}",
rust_shape[2],
golden_shape[2]
);
assert!(
rust_shape[3] >= golden_shape[3],
"Width {} < golden width {}",
rust_shape[3],
golden_shape[3]
);
// Compare pixel values - only compare the actual image region, not padding
let rust_pixels = result.pixel_values_flat();
let golden_pixels_flat: Vec<f32> = golden_pixels.iter().copied().collect();
// Calculate indices for the actual image region (not padding)
let h = golden_shape[2];
let w = golden_shape[3];
let rust_w = rust_shape[3];
let mut max_diff = 0.0f32;
for c in 0..3 {
for y in 0..h {
for x in 0..w {
let golden_idx = c * h * w + y * w + x;
let rust_idx = c * rust_shape[2] * rust_w + y * rust_w + x;
let diff = (rust_pixels[rust_idx] - golden_pixels_flat[golden_idx]).abs();
max_diff = max_diff.max(diff);
}
}
}
println!(
"pixtral - {} image - Max pixel diff: {:.6}",
image_name, max_diff
);
// Allow tolerance for bicubic interpolation differences between PIL and Rust image library
// Pixtral uses bicubic which has larger differences than bilinear
assert!(
max_diff < 0.06,
"Max pixel difference {} exceeds tolerance 0.06 for {}",
max_diff,
image_name
);
}
#[test]
fn test_pixtral_golden_square() {
run_pixtral_golden_test("square");
}
#[test]
fn test_pixtral_golden_tall() {
run_pixtral_golden_test("tall");
}
#[test]
fn test_pixtral_golden_wide() {
run_pixtral_golden_test("wide");
}
#[test]
fn test_pixtral_golden_small() {
run_pixtral_golden_test("small");
}
#[test]
fn test_pixtral_golden_tiny() {
run_pixtral_golden_test("tiny");
}
#[test]
fn test_pixtral_golden_very_tall() {
run_pixtral_golden_test("very_tall");
}
#[test]
fn test_pixtral_golden_very_wide() {
run_pixtral_golden_test("very_wide");
}
#[test]
fn test_pixtral_golden_large() {
run_pixtral_golden_test("large");
}
#[test]
fn test_pixtral_golden_odd_dims() {
run_pixtral_golden_test("odd_dims");
}
#[test]
fn test_pixtral_golden_grayscale() {
run_pixtral_golden_test("grayscale");
}