[model-gateway] add mistral 3 image processor (#14445)

Co-authored-by: Chang Su <chang.s.su@oracle.com>

sgl-router/tests/vision_golden_tests.rs

@@ -19,8 +19,8 @@ use std::{fs::File, io::Read, path::Path};
 use ndarray::{Array4, Array5};
 use sgl_model_gateway::multimodal::vision::{
     image_processor::ModelSpecificValue, ImagePreProcessor, Llama4VisionProcessor, LlavaProcessor,
-    Phi3VisionProcessor, Phi4VisionProcessor, PreProcessorConfig, Qwen2VLProcessor,
-    Qwen3VLProcessor,
+    Phi3VisionProcessor, Phi4VisionProcessor, PixtralProcessor, PreProcessorConfig,
+    Qwen2VLProcessor, Qwen3VLProcessor,
 };
 
 /// Load a numpy .npz file and extract pixel_values
@@ -396,7 +396,7 @@ fn run_qwen2_vl_golden_test(image_name: &str) {
 
     // Verify shapes match
     let expected_num_patches = grid_t * grid_h * grid_w;
-    let patch_size = config.patch_size.unwrap_or(14);
+    let patch_size = config.get_patch_size(14);
     let temporal_patch_size = config.temporal_patch_size.unwrap_or(2);
     let expected_patch_features = 3 * temporal_patch_size * patch_size * patch_size;
 
@@ -580,7 +580,7 @@ fn run_qwen3_vl_golden_test(image_name: &str) {
 
     // Verify shapes match (Qwen3-VL has patch_size=16)
     let expected_num_patches = grid_t * grid_h * grid_w;
-    let patch_size = config.patch_size.unwrap_or(16);
+    let patch_size = config.get_patch_size(16);
     let temporal_patch_size = config.temporal_patch_size.unwrap_or(2);
     let expected_patch_features = 3 * temporal_patch_size * patch_size * patch_size;
 
@@ -1388,3 +1388,214 @@ fn test_llama4_vision_golden_odd_dims() {
 fn test_llama4_vision_golden_grayscale() {
     run_llama4_vision_golden_test("grayscale");
 }
+
+// ============================================================================
+// Pixtral/Mistral3 Vision tests
+// ============================================================================
+
+/// Load image_sizes from npz file for Pixtral
+fn load_pixtral_image_sizes(path: &Path) -> Vec<(usize, usize)> {
+    let file = File::open(path).expect("Failed to open golden file");
+    let mut npz = npyz::npz::NpzArchive::new(file).expect("Failed to parse npz");
+
+    let reader = npz
+        .by_name("image_sizes")
+        .expect("Failed to read npz")
+        .expect("No image_sizes");
+
+    let shape = reader.shape().to_vec();
+
+    // Read data as i64 vec (numpy default for int)
+    let data: Vec<i64> = reader.into_vec().expect("Failed to read array");
+
+    // Convert to Vec<(usize, usize)>
+    let num_images = shape[0] as usize;
+    (0..num_images)
+        .map(|i| (data[i * 2] as usize, data[i * 2 + 1] as usize))
+        .collect()
+}
+
+/// Run a Pixtral golden test for a specific image.
+///
+/// This test validates:
+/// 1. Output shape matches (batch, 3, H, W)
+/// 2. image_sizes match
+/// 3. Pixel values match HuggingFace output
+/// 4. Token count is correct
+///
+/// Pixtral processing:
+/// - Longest edge: 1024 (default)
+/// - Patch size: 16
+/// - Normalization: CLIP mean/std
+/// - No tiling - single output per image
+fn run_pixtral_golden_test(image_name: &str) {
+    let golden_dir = Path::new("tests/fixtures/golden/pixtral");
+    let image_path = Path::new("tests/fixtures/images").join(format!("{}.jpg", image_name));
+
+    if !golden_dir.exists() || !image_path.exists() {
+        eprintln!(
+            "Golden test fixtures for pixtral/{} not found, skipping test",
+            image_name
+        );
+        eprintln!("Run: python scripts/generate_vision_golden.py --model pixtral");
+        return;
+    }
+
+    let npz_path = golden_dir.join(format!("golden_{}.npz", image_name));
+    let config = load_config(&golden_dir.join("preprocessor_config.json"));
+
+    // Load golden values
+    let golden_pixels = load_golden_npz(&npz_path);
+    let golden_shape: Vec<usize> = golden_pixels.shape().to_vec();
+    let golden_image_sizes = load_pixtral_image_sizes(&npz_path);
+    let golden_num_tokens = load_golden_num_tokens(&npz_path);
+
+    // Process image with our Rust processor
+    let image = image::open(&image_path).expect("Failed to open image");
+    let processor = PixtralProcessor::from_preprocessor_config(&config);
+    let result = processor
+        .preprocess(&[image], &config)
+        .expect("Processing failed");
+
+    // Check image_sizes from model_specific
+    let rust_image_sizes: Vec<(usize, usize)> = match result.model_specific.get("image_sizes") {
+        Some(ModelSpecificValue::IntTensor { data, shape }) => {
+            let num_images = shape[0];
+            (0..num_images)
+                .map(|i| (data[i * 2] as usize, data[i * 2 + 1] as usize))
+                .collect()
+        }
+        _ => panic!("Expected image_sizes in model_specific"),
+    };
+
+    println!(
+        "pixtral - {} image - Image sizes: golden={:?}, rust={:?}",
+        image_name, golden_image_sizes, rust_image_sizes
+    );
+    assert_eq!(
+        golden_image_sizes, rust_image_sizes,
+        "image_sizes mismatch for {}",
+        image_name
+    );
+
+    // Check num_tokens
+    let rust_num_tokens = result.num_img_tokens[0];
+    println!(
+        "pixtral - {} image - Tokens: golden={}, rust={}",
+        image_name, golden_num_tokens, rust_num_tokens
+    );
+    assert_eq!(
+        golden_num_tokens, rust_num_tokens,
+        "num_tokens mismatch for {}",
+        image_name
+    );
+
+    // Check output shape
+    let rust_shape = result.pixel_values.shape();
+    println!(
+        "pixtral - {} image - Shape: golden={:?}, rust={:?}",
+        image_name, golden_shape, rust_shape
+    );
+
+    // Pixtral outputs [batch, C, H, W] with padding to max size in batch
+    // Single image should match golden shape exactly
+    assert_eq!(rust_shape[0], 1, "Expected batch dim to be 1");
+    assert_eq!(rust_shape[1], golden_shape[1], "Channel mismatch");
+    assert!(
+        rust_shape[2] >= golden_shape[2],
+        "Height {} < golden height {}",
+        rust_shape[2],
+        golden_shape[2]
+    );
+    assert!(
+        rust_shape[3] >= golden_shape[3],
+        "Width {} < golden width {}",
+        rust_shape[3],
+        golden_shape[3]
+    );
+
+    // Compare pixel values - only compare the actual image region, not padding
+    let rust_pixels = result.pixel_values_flat();
+    let golden_pixels_flat: Vec<f32> = golden_pixels.iter().copied().collect();
+
+    // Calculate indices for the actual image region (not padding)
+    let h = golden_shape[2];
+    let w = golden_shape[3];
+    let rust_w = rust_shape[3];
+
+    let mut max_diff = 0.0f32;
+    for c in 0..3 {
+        for y in 0..h {
+            for x in 0..w {
+                let golden_idx = c * h * w + y * w + x;
+                let rust_idx = c * rust_shape[2] * rust_w + y * rust_w + x;
+                let diff = (rust_pixels[rust_idx] - golden_pixels_flat[golden_idx]).abs();
+                max_diff = max_diff.max(diff);
+            }
+        }
+    }
+
+    println!(
+        "pixtral - {} image - Max pixel diff: {:.6}",
+        image_name, max_diff
+    );
+
+    // Allow tolerance for bicubic interpolation differences between PIL and Rust image library
+    // Pixtral uses bicubic which has larger differences than bilinear
+    assert!(
+        max_diff < 0.06,
+        "Max pixel difference {} exceeds tolerance 0.06 for {}",
+        max_diff,
+        image_name
+    );
+}
+
+#[test]
+fn test_pixtral_golden_square() {
+    run_pixtral_golden_test("square");
+}
+
+#[test]
+fn test_pixtral_golden_tall() {
+    run_pixtral_golden_test("tall");
+}
+
+#[test]
+fn test_pixtral_golden_wide() {
+    run_pixtral_golden_test("wide");
+}
+
+#[test]
+fn test_pixtral_golden_small() {
+    run_pixtral_golden_test("small");
+}
+
+#[test]
+fn test_pixtral_golden_tiny() {
+    run_pixtral_golden_test("tiny");
+}
+
+#[test]
+fn test_pixtral_golden_very_tall() {
+    run_pixtral_golden_test("very_tall");
+}
+
+#[test]
+fn test_pixtral_golden_very_wide() {
+    run_pixtral_golden_test("very_wide");
+}
+
+#[test]
+fn test_pixtral_golden_large() {
+    run_pixtral_golden_test("large");
+}
+
+#[test]
+fn test_pixtral_golden_odd_dims() {
+    run_pixtral_golden_test("odd_dims");
+}
+
+#[test]
+fn test_pixtral_golden_grayscale() {
+    run_pixtral_golden_test("grayscale");
+}