//! Golden tests for vision processors. //! //! These tests compare Rust preprocessor output against golden outputs //! generated by HuggingFace transformers to ensure pixel-perfect compatibility. //! //! Modes tested: //! - `llava/` - Standard CLIP processing (llava-hf/* models, no expand-to-square) //! - `llava_pad/` - Expand-to-square mode (liuhaotian/llava-* models, image_aspect_ratio=pad) //! - `qwen2_vl/` - Dynamic resolution with smart resize (Qwen/Qwen2-VL-* models) //! - `qwen3_vl/` - Dynamic resolution with patch_size=16 and [0.5,0.5,0.5] norm (Qwen/Qwen3-VL-* models) //! //! To regenerate golden outputs: //! ```bash //! python scripts/generate_vision_golden.py //! ``` use std::{fs::File, io::Read, path::Path}; use ndarray::{Array4, Array5}; use sgl_model_gateway::multimodal::vision::{ image_processor::ModelSpecificValue, ImagePreProcessor, Llama4VisionProcessor, LlavaProcessor, Phi3VisionProcessor, Phi4VisionProcessor, PixtralProcessor, PreProcessorConfig, Qwen2VLProcessor, Qwen3VLProcessor, }; /// Load a numpy .npz file and extract pixel_values fn load_golden_npz(path: &Path) -> Array4 { let file = File::open(path).expect("Failed to open golden file"); let mut npz = npyz::npz::NpzArchive::new(file).expect("Failed to parse npz"); // Read pixel_values array (npz stores arrays without .npy extension in the lookup) let reader = npz .by_name("pixel_values") .expect("Failed to read npz") .expect("No pixel_values"); // Get shape from npy header let shape = reader.shape().to_vec(); assert_eq!(shape.len(), 4, "Expected 4D tensor [B, C, H, W]"); // Read data as f32 vec let data: Vec = reader.into_vec().expect("Failed to read array"); // Convert to Array4 Array4::from_shape_vec( ( shape[0] as usize, shape[1] as usize, shape[2] as usize, shape[3] as usize, ), data, ) .expect("Shape conversion failed") } /// Load preprocessor config from JSON fn load_config(path: &Path) -> PreProcessorConfig { let mut file = File::open(path).expect("Failed to open config"); let mut contents = String::new(); file.read_to_string(&mut contents) .expect("Failed to read config"); PreProcessorConfig::from_json(&contents).expect("Failed to parse config") } /// Compare two 4D tensors and return max absolute difference fn max_diff(a: &Array4, b: &ndarray::ArrayD) -> f32 { assert_eq!(a.shape(), b.shape(), "Shape mismatch"); // Convert ArrayD to Array4 for comparison let b_4d = b .clone() .into_dimensionality::() .expect("Expected 4D tensor"); (a - &b_4d) .mapv(|v| v.abs()) .fold(0.0f32, |acc, &v| acc.max(v)) } /// Load image_grid_thw from npz file fn load_golden_grid_thw(path: &Path) -> Vec { let file = File::open(path).expect("Failed to open golden file"); let mut npz = npyz::npz::NpzArchive::new(file).expect("Failed to parse npz"); let reader = npz .by_name("image_grid_thw") .expect("Failed to read npz") .expect("No image_grid_thw"); // Shape not needed, data is flat let _shape = reader.shape(); // Read data as i64 vec (numpy default for int) let data: Vec = reader.into_vec().expect("Failed to read array"); // Convert to u32 data.into_iter().map(|v| v as u32).collect() } /// Load num_tokens from npz file fn load_golden_num_tokens(path: &Path) -> usize { let file = File::open(path).expect("Failed to open golden file"); let mut npz = npyz::npz::NpzArchive::new(file).expect("Failed to parse npz"); let reader = npz .by_name("num_tokens") .expect("Failed to read npz") .expect("No num_tokens"); // Read single value as i64 let data: Vec = reader.into_vec().expect("Failed to read array"); data[0] as usize } /// Run a golden test for a specific mode and image. /// /// # Arguments /// * `mode` - Either "llava" (standard CLIP) or "llava_pad" (expand-to-square mode) /// * `image_name` - Name of the test image (e.g., "square", "tall", "wide", "small") fn run_golden_test(mode: &str, image_name: &str) { let golden_dir = Path::new("tests/fixtures/golden").join(mode); let image_path = Path::new("tests/fixtures/images").join(format!("{}.jpg", image_name)); if !golden_dir.exists() || !image_path.exists() { eprintln!( "Golden test fixtures for {}/{} not found, skipping test", mode, image_name ); eprintln!("Run: python scripts/generate_vision_golden.py"); return; } let golden = load_golden_npz(&golden_dir.join(format!("golden_{}.npz", image_name))); let config = load_config(&golden_dir.join("preprocessor_config.json")); let image = image::open(&image_path).expect("Failed to open image"); let processor: Box = match mode { "llava" => Box::new(LlavaProcessor::new()), "llava_pad" => Box::new(LlavaProcessor::new_with_pad()), _ => panic!("Unknown test mode: {}", mode), }; let result = processor .preprocess(&[image], &config) .expect("Processing failed"); let diff = max_diff(&golden, &result.pixel_values); println!( "{} - {} image - Max difference: {:.6}", mode, image_name, diff ); println!("Golden shape: {:?}", golden.shape()); println!("Rust shape: {:?}", result.pixel_values.shape()); // Allow tolerance for floating point and interpolation algorithm differences // Different interpolation implementations (Rust vs Python/PIL) can produce // small numerical differences, especially for edge cases like tiny or extreme // aspect ratio images assert!(diff < 0.1, "Max difference {} exceeds tolerance 0.1", diff); } // ============================================================================ // Standard CLIP mode tests (llava-hf/* models, no expand-to-square) // ============================================================================ #[test] fn test_llava_golden_square() { run_golden_test("llava", "square"); } #[test] fn test_llava_golden_tall() { run_golden_test("llava", "tall"); } #[test] fn test_llava_golden_wide() { run_golden_test("llava", "wide"); } #[test] fn test_llava_golden_small() { run_golden_test("llava", "small"); } #[test] fn test_llava_golden_tiny() { run_golden_test("llava", "tiny"); } #[test] fn test_llava_golden_very_tall() { run_golden_test("llava", "very_tall"); } #[test] fn test_llava_golden_very_wide() { run_golden_test("llava", "very_wide"); } #[test] fn test_llava_golden_large() { run_golden_test("llava", "large"); } #[test] fn test_llava_golden_odd_dims() { run_golden_test("llava", "odd_dims"); } #[test] fn test_llava_golden_grayscale() { run_golden_test("llava", "grayscale"); } // ============================================================================ // Pad mode tests (liuhaotian/llava-* models, image_aspect_ratio=pad) // ============================================================================ #[test] fn test_llava_pad_golden_square() { run_golden_test("llava_pad", "square"); } #[test] fn test_llava_pad_golden_tall() { run_golden_test("llava_pad", "tall"); } #[test] fn test_llava_pad_golden_wide() { run_golden_test("llava_pad", "wide"); } #[test] fn test_llava_pad_golden_small() { run_golden_test("llava_pad", "small"); } #[test] fn test_llava_pad_golden_tiny() { run_golden_test("llava_pad", "tiny"); } #[test] fn test_llava_pad_golden_very_tall() { run_golden_test("llava_pad", "very_tall"); } #[test] fn test_llava_pad_golden_very_wide() { run_golden_test("llava_pad", "very_wide"); } #[test] fn test_llava_pad_golden_large() { run_golden_test("llava_pad", "large"); } #[test] fn test_llava_pad_golden_odd_dims() { run_golden_test("llava_pad", "odd_dims"); } #[test] fn test_llava_pad_golden_grayscale() { run_golden_test("llava_pad", "grayscale"); } // ============================================================================ // Token count tests // ============================================================================ #[test] fn test_llava_token_count() { let golden_dir = Path::new("tests/fixtures/golden/llava"); if !golden_dir.exists() { eprintln!("Golden test fixtures not found, skipping test"); return; } let config = load_config(&golden_dir.join("preprocessor_config.json")); let processor = LlavaProcessor::new(); // LLaVA 1.5 with 336x336 and patch_size=14: (336/14)^2 = 576 tokens let tokens = processor.calculate_num_tokens(336, 336, &config); assert_eq!( tokens, 576, "Expected 576 tokens for 336x336 with patch_size=14" ); } // ============================================================================ // Qwen2-VL tests // ============================================================================ /// Load flattened pixel values from Qwen2-VL npz file. /// Returns (data, shape) where shape is (num_patches, patch_features). fn load_golden_qwen2_vl_pixels(path: &Path) -> (Vec, (usize, usize)) { let file = File::open(path).expect("Failed to open golden file"); let mut npz = npyz::npz::NpzArchive::new(file).expect("Failed to parse npz"); let reader = npz .by_name("pixel_values") .expect("Failed to read npz") .expect("No pixel_values"); let shape = reader.shape().to_vec(); assert_eq!(shape.len(), 2, "Expected 2D tensor for Qwen2-VL patches"); let data: Vec = reader.into_vec().expect("Failed to read array"); (data, (shape[0] as usize, shape[1] as usize)) } /// Run a Qwen2-VL golden test for a specific image. /// /// This test validates: /// 1. image_grid_thw matches the HuggingFace output /// 2. num_tokens calculation is correct /// 3. Pixel values match after reshaping to patch format fn run_qwen2_vl_golden_test(image_name: &str) { let golden_dir = Path::new("tests/fixtures/golden/qwen2_vl"); let image_path = Path::new("tests/fixtures/images").join(format!("{}.jpg", image_name)); if !golden_dir.exists() || !image_path.exists() { eprintln!( "Golden test fixtures for qwen2_vl/{} not found, skipping test", image_name ); eprintln!("Run: python scripts/generate_vision_golden.py --model qwen2_vl"); return; } let npz_path = golden_dir.join(format!("golden_{}.npz", image_name)); let config = load_config(&golden_dir.join("preprocessor_config.json")); // Load golden values let golden_grid_thw = load_golden_grid_thw(&npz_path); let golden_num_tokens = load_golden_num_tokens(&npz_path); let (golden_pixels, golden_shape) = load_golden_qwen2_vl_pixels(&npz_path); // Process image with our Rust processor let image = image::open(&image_path).expect("Failed to open image"); let processor = Qwen2VLProcessor::from_preprocessor_config(&config); let result = processor .preprocess(&[image], &config) .expect("Processing failed"); // Extract image_grid_thw from result let rust_grid_thw = match result.model_specific.get("image_grid_thw") { Some(ModelSpecificValue::UintTensor { data, shape }) => { assert_eq!(shape, &[1, 3], "Expected shape [1, 3] for single image"); data.clone() } _ => panic!("Expected image_grid_thw in model_specific"), }; // Compare grid dimensions println!( "qwen2_vl - {} image - Grid T H W: golden={:?}, rust={:?}", image_name, golden_grid_thw, rust_grid_thw ); assert_eq!( golden_grid_thw, rust_grid_thw, "image_grid_thw mismatch for {}", image_name ); // Compare token counts let rust_num_tokens = result.num_img_tokens[0]; println!( "qwen2_vl - {} image - Tokens: golden={}, rust={}", image_name, golden_num_tokens, rust_num_tokens ); assert_eq!( golden_num_tokens, rust_num_tokens, "num_tokens mismatch for {}", image_name ); // Compare pixel values by reshaping our output to patch format let grid_t = rust_grid_thw[0] as usize; let grid_h = rust_grid_thw[1] as usize; let grid_w = rust_grid_thw[2] as usize; // Get the tensor for the first image (batch index 0) let pixel_values = &result.pixel_values; let tensor_3d_dyn = pixel_values.index_axis(ndarray::Axis(0), 0).to_owned(); let tensor_3d = tensor_3d_dyn .into_dimensionality::() .expect("Expected 3D tensor for Qwen2-VL"); // Reshape to patches format let rust_patches = processor.reshape_to_patches(&tensor_3d, grid_t, grid_h, grid_w); // Verify shapes match let expected_num_patches = grid_t * grid_h * grid_w; let patch_size = config.get_patch_size(14); let temporal_patch_size = config.temporal_patch_size.unwrap_or(2); let expected_patch_features = 3 * temporal_patch_size * patch_size * patch_size; println!( "qwen2_vl - {} image - Patch shape: golden={:?}, rust=({}, {})", image_name, golden_shape, expected_num_patches, expected_patch_features ); assert_eq!( golden_shape, (expected_num_patches, expected_patch_features), "Patch shape mismatch" ); assert_eq!( rust_patches.len(), expected_num_patches * expected_patch_features, "Rust patches size mismatch" ); // Compare pixel values let max_diff = rust_patches .iter() .zip(golden_pixels.iter()) .map(|(r, g)| (r - g).abs()) .fold(0.0f32, f32::max); println!( "qwen2_vl - {} image - Max pixel diff: {:.6}", image_name, max_diff ); // Allow tolerance for floating point and interpolation differences // Different interpolation implementations (Rust vs Python/PIL) can produce // small numerical differences, especially for edge cases assert!( max_diff < 0.1, "Max pixel difference {} exceeds tolerance 0.1 for {}", max_diff, image_name ); } #[test] fn test_qwen2_vl_golden_square() { run_qwen2_vl_golden_test("square"); } #[test] fn test_qwen2_vl_golden_tall() { run_qwen2_vl_golden_test("tall"); } #[test] fn test_qwen2_vl_golden_wide() { run_qwen2_vl_golden_test("wide"); } #[test] fn test_qwen2_vl_golden_small() { run_qwen2_vl_golden_test("small"); } #[test] fn test_qwen2_vl_golden_tiny() { run_qwen2_vl_golden_test("tiny"); } #[test] fn test_qwen2_vl_golden_very_tall() { run_qwen2_vl_golden_test("very_tall"); } #[test] fn test_qwen2_vl_golden_very_wide() { run_qwen2_vl_golden_test("very_wide"); } #[test] fn test_qwen2_vl_golden_large() { run_qwen2_vl_golden_test("large"); } #[test] fn test_qwen2_vl_golden_odd_dims() { run_qwen2_vl_golden_test("odd_dims"); } #[test] fn test_qwen2_vl_golden_grayscale() { run_qwen2_vl_golden_test("grayscale"); } // ============================================================================ // Qwen3-VL tests // ============================================================================ /// Run a Qwen3-VL golden test for a specific image. /// /// This test validates: /// 1. image_grid_thw matches the HuggingFace output /// 2. num_tokens calculation is correct /// 3. Pixel values match after reshaping to patch format /// /// Key differences from Qwen2-VL: /// - patch_size: 16 (vs 14) /// - factor: 32 (vs 28) /// - normalization: [0.5, 0.5, 0.5] (vs CLIP) fn run_qwen3_vl_golden_test(image_name: &str) { let golden_dir = Path::new("tests/fixtures/golden/qwen3_vl"); let image_path = Path::new("tests/fixtures/images").join(format!("{}.jpg", image_name)); if !golden_dir.exists() || !image_path.exists() { eprintln!( "Golden test fixtures for qwen3_vl/{} not found, skipping test", image_name ); eprintln!("Run: python scripts/generate_vision_golden.py --model qwen3_vl"); return; } let npz_path = golden_dir.join(format!("golden_{}.npz", image_name)); let config = load_config(&golden_dir.join("preprocessor_config.json")); // Load golden values let golden_grid_thw = load_golden_grid_thw(&npz_path); let golden_num_tokens = load_golden_num_tokens(&npz_path); let (golden_pixels, golden_shape) = load_golden_qwen2_vl_pixels(&npz_path); // Process image with our Rust processor let image = image::open(&image_path).expect("Failed to open image"); let processor = Qwen3VLProcessor::from_preprocessor_config(&config); let result = processor .preprocess(&[image], &config) .expect("Processing failed"); // Extract image_grid_thw from result let rust_grid_thw = match result.model_specific.get("image_grid_thw") { Some(ModelSpecificValue::UintTensor { data, shape }) => { assert_eq!(shape, &[1, 3], "Expected shape [1, 3] for single image"); data.clone() } _ => panic!("Expected image_grid_thw in model_specific"), }; // Compare grid dimensions println!( "qwen3_vl - {} image - Grid T H W: golden={:?}, rust={:?}", image_name, golden_grid_thw, rust_grid_thw ); assert_eq!( golden_grid_thw, rust_grid_thw, "image_grid_thw mismatch for {}", image_name ); // Compare token counts let rust_num_tokens = result.num_img_tokens[0]; println!( "qwen3_vl - {} image - Tokens: golden={}, rust={}", image_name, golden_num_tokens, rust_num_tokens ); assert_eq!( golden_num_tokens, rust_num_tokens, "num_tokens mismatch for {}", image_name ); // Compare pixel values by reshaping our output to patch format let grid_t = rust_grid_thw[0] as usize; let grid_h = rust_grid_thw[1] as usize; let grid_w = rust_grid_thw[2] as usize; // Get the tensor for the first image (batch index 0) let pixel_values = &result.pixel_values; let tensor_3d_dyn = pixel_values.index_axis(ndarray::Axis(0), 0).to_owned(); let tensor_3d = tensor_3d_dyn .into_dimensionality::() .expect("Expected 3D tensor for Qwen3-VL"); // Reshape to patches format let rust_patches = processor.reshape_to_patches(&tensor_3d, grid_t, grid_h, grid_w); // Verify shapes match (Qwen3-VL has patch_size=16) let expected_num_patches = grid_t * grid_h * grid_w; let patch_size = config.get_patch_size(16); let temporal_patch_size = config.temporal_patch_size.unwrap_or(2); let expected_patch_features = 3 * temporal_patch_size * patch_size * patch_size; println!( "qwen3_vl - {} image - Patch shape: golden={:?}, rust=({}, {})", image_name, golden_shape, expected_num_patches, expected_patch_features ); assert_eq!( golden_shape, (expected_num_patches, expected_patch_features), "Patch shape mismatch" ); assert_eq!( rust_patches.len(), expected_num_patches * expected_patch_features, "Rust patches size mismatch" ); // Compare pixel values let max_diff = rust_patches .iter() .zip(golden_pixels.iter()) .map(|(r, g)| (r - g).abs()) .fold(0.0f32, f32::max); println!( "qwen3_vl - {} image - Max pixel diff: {:.6}", image_name, max_diff ); // Allow tolerance for floating point and interpolation differences // Max diff is ~0.03 due to resize interpolation differences between Rust and HuggingFace assert!( max_diff < 0.05, "Max pixel difference {} exceeds tolerance 0.05 for {}", max_diff, image_name ); } #[test] fn test_qwen3_vl_golden_square() { run_qwen3_vl_golden_test("square"); } #[test] fn test_qwen3_vl_golden_tall() { run_qwen3_vl_golden_test("tall"); } #[test] fn test_qwen3_vl_golden_wide() { run_qwen3_vl_golden_test("wide"); } #[test] fn test_qwen3_vl_golden_small() { run_qwen3_vl_golden_test("small"); } #[test] fn test_qwen3_vl_golden_tiny() { run_qwen3_vl_golden_test("tiny"); } #[test] fn test_qwen3_vl_golden_very_tall() { run_qwen3_vl_golden_test("very_tall"); } #[test] fn test_qwen3_vl_golden_very_wide() { run_qwen3_vl_golden_test("very_wide"); } #[test] fn test_qwen3_vl_golden_large() { run_qwen3_vl_golden_test("large"); } #[test] fn test_qwen3_vl_golden_odd_dims() { run_qwen3_vl_golden_test("odd_dims"); } #[test] fn test_qwen3_vl_golden_grayscale() { run_qwen3_vl_golden_test("grayscale"); } // ============================================================================ // Phi3-Vision tests // ============================================================================ /// Load a 5D numpy .npz file for Phi3-Vision (batch, num_crops+1, C, H, W) fn load_golden_npz_5d(path: &Path) -> Array5 { let file = File::open(path).expect("Failed to open golden file"); let mut npz = npyz::npz::NpzArchive::new(file).expect("Failed to parse npz"); let reader = npz .by_name("pixel_values") .expect("Failed to read npz") .expect("No pixel_values"); let shape = reader.shape().to_vec(); assert_eq!(shape.len(), 5, "Expected 5D tensor [B, N, C, H, W]"); let data: Vec = reader.into_vec().expect("Failed to read array"); Array5::from_shape_vec( ( shape[0] as usize, shape[1] as usize, shape[2] as usize, shape[3] as usize, shape[4] as usize, ), data, ) .expect("Shape conversion failed") } /// Load image_sizes from Phi3-Vision npz file (2D tensor [batch, 2]) fn load_phi3_image_sizes(path: &Path) -> Vec<(u32, u32)> { let file = File::open(path).expect("Failed to open golden file"); let mut npz = npyz::npz::NpzArchive::new(file).expect("Failed to parse npz"); let reader = npz .by_name("image_sizes") .expect("Failed to read npz") .expect("No image_sizes"); let shape = reader.shape().to_vec(); let data: Vec = reader.into_vec().expect("Failed to read array"); // Reshape to pairs let num_images = shape[0] as usize; (0..num_images) .map(|i| (data[i * 2] as u32, data[i * 2 + 1] as u32)) .collect() } /// Load num_img_tokens from Phi3-Vision npz file fn load_phi3_num_img_tokens(path: &Path) -> Vec { let file = File::open(path).expect("Failed to open golden file"); let mut npz = npyz::npz::NpzArchive::new(file).expect("Failed to parse npz"); let reader = npz .by_name("num_img_tokens") .expect("Failed to read npz") .expect("No num_img_tokens"); let data: Vec = reader.into_vec().expect("Failed to read array"); data.into_iter().map(|v| v as usize).collect() } /// Compare two 5D tensors and return max absolute difference fn max_diff_5d(a: &Array5, b: &Array5) -> f32 { assert_eq!(a.shape(), b.shape(), "Shape mismatch"); (a - b).mapv(|v| v.abs()).fold(0.0f32, |acc, &v| acc.max(v)) } /// Find the location and value of max difference between two 5D tensors #[allow(dead_code)] fn find_max_diff_location_5d( golden: &Array5, rust: &Array5, image_name: &str, ) -> (f32, (usize, usize, usize, usize, usize)) { assert_eq!(golden.shape(), rust.shape(), "Shape mismatch"); let diff = (golden - rust).mapv(|v| v.abs()); let mut max_diff = 0.0f32; let mut max_pos = (0, 0, 0, 0, 0); // Find per-tile max differences for b in 0..golden.shape()[0] { for t in 0..golden.shape()[1] { let tile_diff = diff.slice(ndarray::s![b, t, .., .., ..]); let tile_max = tile_diff.fold(0.0f32, |acc, &v| acc.max(v)); if tile_max > 0.1 { let golden_tile = golden.slice(ndarray::s![b, t, .., .., ..]); let rust_tile = rust.slice(ndarray::s![b, t, .., .., ..]); println!( " {} tile {}: diff={:.4}, golden_range=[{:.4}, {:.4}], rust_range=[{:.4}, {:.4}]", image_name, t, tile_max, golden_tile.fold(f32::MAX, |a, &v| a.min(v)), golden_tile.fold(f32::MIN, |a, &v| a.max(v)), rust_tile.fold(f32::MAX, |a, &v| a.min(v)), rust_tile.fold(f32::MIN, |a, &v| a.max(v)) ); } if tile_max > max_diff { max_diff = tile_max; // Find exact position for c in 0..golden.shape()[2] { for h in 0..golden.shape()[3] { for w in 0..golden.shape()[4] { if diff[[b, t, c, h, w]] == max_diff { max_pos = (b, t, c, h, w); } } } } } } } (max_diff, max_pos) } /// Run a Phi3-Vision golden test for a specific image. /// /// This test validates: /// 1. Output shape is [1, num_crops+1, 3, 336, 336] /// 2. image_sizes matches HuggingFace output /// 3. num_img_tokens matches HuggingFace output /// 4. Pixel values match within tolerance fn run_phi3_vision_golden_test(image_name: &str) { let golden_dir = Path::new("tests/fixtures/golden/phi3_vision"); let image_path = Path::new("tests/fixtures/images").join(format!("{}.jpg", image_name)); if !golden_dir.exists() || !image_path.exists() { eprintln!( "Golden test fixtures for phi3_vision/{} not found, skipping test", image_name ); eprintln!("Run: python scripts/generate_vision_golden.py --model phi3_vision"); return; } let npz_path = golden_dir.join(format!("golden_{}.npz", image_name)); let config = load_config(&golden_dir.join("preprocessor_config.json")); // Load golden values let golden_pixels = load_golden_npz_5d(&npz_path); let golden_image_sizes = load_phi3_image_sizes(&npz_path); let golden_num_tokens = load_phi3_num_img_tokens(&npz_path); // Process image with our Rust processor let image = image::open(&image_path).expect("Failed to open image"); let processor = Phi3VisionProcessor::from_preprocessor_config(&config); let result = processor .preprocess(&[image], &config) .expect("Processing failed"); // Check output shape let rust_shape = result.pixel_values.shape(); let golden_shape = golden_pixels.shape(); println!( "phi3_vision - {} image - Shape: golden={:?}, rust={:?}", image_name, golden_shape, rust_shape ); assert_eq!( rust_shape, golden_shape, "Shape mismatch for phi3_vision/{}", image_name ); // Check image_sizes // Note: HuggingFace returns [h, w], we store as (w, h) but model_specific stores (h, w) let rust_image_sizes: Vec<(u32, u32)> = match result.model_specific.get("image_sizes") { Some(ModelSpecificValue::UintTensor { data, shape }) => { let num_images = shape[0]; (0..num_images) .map(|i| (data[i * 2], data[i * 2 + 1])) .collect() } _ => panic!("Expected image_sizes in model_specific"), }; println!( "phi3_vision - {} image - Image sizes (h, w): golden={:?}, rust={:?}", image_name, golden_image_sizes, rust_image_sizes ); assert_eq!( golden_image_sizes, rust_image_sizes, "image_sizes mismatch for {}", image_name ); // Check num_img_tokens println!( "phi3_vision - {} image - Num tokens: golden={:?}, rust={:?}", image_name, golden_num_tokens, result.num_img_tokens ); assert_eq!( golden_num_tokens, result.num_img_tokens, "num_img_tokens mismatch for {}", image_name ); // Compare pixel values // Convert rust ArrayD to Array5 for comparison let rust_pixels = result .pixel_values .clone() .into_dimensionality::() .expect("Failed to convert to Ix5"); let pixel_diff = max_diff_5d(&golden_pixels, &rust_pixels); println!( "phi3_vision - {} image - Max pixel diff: {:.6}", image_name, pixel_diff ); // If there's a large difference, print detailed info if pixel_diff > 0.1 { let (max_diff, max_pos) = find_max_diff_location_5d(&golden_pixels, &rust_pixels, image_name); println!( "phi3_vision - {} image - Max diff {:.4} at position {:?}", image_name, max_diff, max_pos ); let (b, t, c, h, w) = max_pos; println!( " golden value: {:.4}, rust value: {:.4}", golden_pixels[[b, t, c, h, w]], rust_pixels[[b, t, c, h, w]] ); } // Allow tolerance for floating point and interpolation differences // Using bicubic for global image and bilinear for HD resize to match HuggingFace. assert!( pixel_diff < 0.08, "Max pixel difference {} exceeds tolerance 0.08 for {}", pixel_diff, image_name ); } #[test] fn test_phi3_vision_golden_square() { run_phi3_vision_golden_test("square"); } #[test] fn test_phi3_vision_golden_tall() { run_phi3_vision_golden_test("tall"); } #[test] fn test_phi3_vision_golden_wide() { run_phi3_vision_golden_test("wide"); } #[test] fn test_phi3_vision_golden_small() { run_phi3_vision_golden_test("small"); } #[test] fn test_phi3_vision_golden_tiny() { run_phi3_vision_golden_test("tiny"); } #[test] fn test_phi3_vision_golden_very_tall() { run_phi3_vision_golden_test("very_tall"); } #[test] fn test_phi3_vision_golden_very_wide() { run_phi3_vision_golden_test("very_wide"); } #[test] fn test_phi3_vision_golden_large() { run_phi3_vision_golden_test("large"); } #[test] fn test_phi3_vision_golden_odd_dims() { run_phi3_vision_golden_test("odd_dims"); } #[test] fn test_phi3_vision_golden_grayscale() { run_phi3_vision_golden_test("grayscale"); } // ============================================================================ // Phi4-Vision tests // ============================================================================ /// Load num_img_tokens from Phi4-Vision npz file fn load_phi4_num_img_tokens(path: &Path) -> Vec { let file = File::open(path).expect("Failed to open golden file"); let mut npz = npyz::npz::NpzArchive::new(file).expect("Failed to parse npz"); let reader = npz .by_name("num_img_tokens") .expect("Failed to read npz") .expect("No num_img_tokens"); let data: Vec = reader.into_vec().expect("Failed to read array"); data.into_iter().map(|v| v as usize).collect() } /// Load image_sizes from Phi4-Vision npz file (2D tensor [batch, 2]) fn load_phi4_image_sizes(path: &Path) -> Vec<(u32, u32)> { let file = File::open(path).expect("Failed to open golden file"); let mut npz = npyz::npz::NpzArchive::new(file).expect("Failed to parse npz"); let reader = npz .by_name("image_sizes") .expect("Failed to read npz") .expect("No image_sizes"); let shape = reader.shape().to_vec(); let data: Vec = reader.into_vec().expect("Failed to read array"); // Reshape to pairs let num_images = shape[0] as usize; (0..num_images) .map(|i| (data[i * 2] as u32, data[i * 2 + 1] as u32)) .collect() } /// Run a Phi4-Vision golden test for a specific image. /// /// This test validates: /// 1. Output shape is [1, num_crops+1, 3, 448, 448] (note: 448 base resolution) /// 2. image_sizes matches HuggingFace output /// 3. num_img_tokens matches HuggingFace output /// 4. Pixel values match within tolerance /// /// Key differences from Phi3-Vision: /// - Base resolution: 448 (vs 336) /// - Normalization: [0.5, 0.5, 0.5] (vs CLIP) /// - Default dynamic_hd: 36 (vs 16) fn run_phi4_vision_golden_test(image_name: &str) { let golden_dir = Path::new("tests/fixtures/golden/phi4_vision"); let image_path = Path::new("tests/fixtures/images").join(format!("{}.jpg", image_name)); if !golden_dir.exists() || !image_path.exists() { eprintln!( "Golden test fixtures for phi4_vision/{} not found, skipping test", image_name ); eprintln!("Run: python scripts/generate_vision_golden.py --model phi4_vision"); return; } let npz_path = golden_dir.join(format!("golden_{}.npz", image_name)); let config = load_config(&golden_dir.join("preprocessor_config.json")); // Load golden values let golden_pixels = load_golden_npz_5d(&npz_path); let golden_image_sizes = load_phi4_image_sizes(&npz_path); let golden_num_tokens = load_phi4_num_img_tokens(&npz_path); // Process image with our Rust processor let image = image::open(&image_path).expect("Failed to open image"); let processor = Phi4VisionProcessor::from_preprocessor_config(&config); let result = processor .preprocess(&[image], &config) .expect("Processing failed"); // Check output shape let rust_shape = result.pixel_values.shape(); let golden_shape = golden_pixels.shape(); println!( "phi4_vision - {} image - Shape: golden={:?}, rust={:?}", image_name, golden_shape, rust_shape ); assert_eq!( rust_shape, golden_shape, "Shape mismatch for phi4_vision/{}", image_name ); // Check image_sizes let rust_image_sizes: Vec<(u32, u32)> = match result.model_specific.get("image_sizes") { Some(ModelSpecificValue::UintTensor { data, shape }) => { let num_images = shape[0]; (0..num_images) .map(|i| (data[i * 2], data[i * 2 + 1])) .collect() } _ => panic!("Expected image_sizes in model_specific"), }; println!( "phi4_vision - {} image - Image sizes (h, w): golden={:?}, rust={:?}", image_name, golden_image_sizes, rust_image_sizes ); assert_eq!( golden_image_sizes, rust_image_sizes, "image_sizes mismatch for {}", image_name ); // Check num_img_tokens println!( "phi4_vision - {} image - Num tokens: golden={:?}, rust={:?}", image_name, golden_num_tokens, result.num_img_tokens ); assert_eq!( golden_num_tokens, result.num_img_tokens, "num_img_tokens mismatch for {}", image_name ); // Compare pixel values let rust_pixels = result .pixel_values .clone() .into_dimensionality::() .expect("Failed to convert to Ix5"); let pixel_diff = max_diff_5d(&golden_pixels, &rust_pixels); println!( "phi4_vision - {} image - Max pixel diff: {:.6}", image_name, pixel_diff ); // If there's a large difference, print detailed info if pixel_diff > 0.1 { let (max_diff, max_pos) = find_max_diff_location_5d(&golden_pixels, &rust_pixels, image_name); println!( "phi4_vision - {} image - Max diff {:.4} at position {:?}", image_name, max_diff, max_pos ); let (b, t, c, h, w) = max_pos; println!( " golden value: {:.4}, rust value: {:.4}", golden_pixels[[b, t, c, h, w]], rust_pixels[[b, t, c, h, w]] ); } // Allow tolerance for floating point and interpolation differences // Using bilinear for HD resize and bicubic for global image to match HuggingFace. assert!( pixel_diff < 0.05, "Max pixel difference {} exceeds tolerance 0.05 for {}", pixel_diff, image_name ); } #[test] fn test_phi4_vision_golden_square() { run_phi4_vision_golden_test("square"); } #[test] fn test_phi4_vision_golden_tall() { run_phi4_vision_golden_test("tall"); } #[test] fn test_phi4_vision_golden_wide() { run_phi4_vision_golden_test("wide"); } #[test] fn test_phi4_vision_golden_small() { run_phi4_vision_golden_test("small"); } #[test] fn test_phi4_vision_golden_tiny() { run_phi4_vision_golden_test("tiny"); } #[test] fn test_phi4_vision_golden_very_tall() { run_phi4_vision_golden_test("very_tall"); } #[test] fn test_phi4_vision_golden_very_wide() { run_phi4_vision_golden_test("very_wide"); } #[test] fn test_phi4_vision_golden_large() { run_phi4_vision_golden_test("large"); } #[test] fn test_phi4_vision_golden_odd_dims() { run_phi4_vision_golden_test("odd_dims"); } #[test] fn test_phi4_vision_golden_grayscale() { run_phi4_vision_golden_test("grayscale"); } // ============================================================================ // LLaMA 4 Vision tests // ============================================================================ /// Load aspect_ratios from npz file for LLaMA 4 fn load_llama4_aspect_ratios(path: &Path) -> Vec<(u32, u32)> { let file = File::open(path).expect("Failed to open golden file"); let mut npz = npyz::npz::NpzArchive::new(file).expect("Failed to parse npz"); let reader = npz .by_name("aspect_ratios") .expect("Failed to read npz") .expect("No aspect_ratios"); let shape = reader.shape().to_vec(); // Read data as i64 vec (numpy default for int) let data: Vec = reader.into_vec().expect("Failed to read array"); // Convert to Vec<(u32, u32)> let num_images = shape[0] as usize; (0..num_images) .map(|i| (data[i * 2] as u32, data[i * 2 + 1] as u32)) .collect() } /// Load pixel_values for LLaMA 4 Vision (3D: [num_tiles, C, H, W]) fn load_llama4_pixels(path: &Path) -> (Vec, Vec) { let file = File::open(path).expect("Failed to open golden file"); let mut npz = npyz::npz::NpzArchive::new(file).expect("Failed to parse npz"); let reader = npz .by_name("pixel_values") .expect("Failed to read npz") .expect("No pixel_values"); let shape: Vec = reader.shape().iter().map(|&s| s as usize).collect(); let data: Vec = reader.into_vec().expect("Failed to read array"); (data, shape) } /// Run a LLaMA 4 Vision golden test for a specific image. /// /// This test validates: /// 1. Output shape matches (batch, num_tiles, 3, 336, 336) /// 2. aspect_ratios match (h_tiles, w_tiles) /// 3. Pixel values match HuggingFace output /// 4. Token count is correct /// /// LLaMA 4 Vision processing: /// - Tile size: 336x336 /// - Max patches: 16 (default) /// - Normalization: [0.5, 0.5, 0.5] mean/std /// - Global tile added when num_tiles > 1 fn run_llama4_vision_golden_test(image_name: &str) { let golden_dir = Path::new("tests/fixtures/golden/llama4_vision"); let image_path = Path::new("tests/fixtures/images").join(format!("{}.jpg", image_name)); if !golden_dir.exists() || !image_path.exists() { eprintln!( "Golden test fixtures for llama4_vision/{} not found, skipping test", image_name ); eprintln!("Run: python scripts/generate_vision_golden.py --model llama4_vision"); return; } let npz_path = golden_dir.join(format!("golden_{}.npz", image_name)); let config = load_config(&golden_dir.join("preprocessor_config.json")); // Load golden values let (golden_pixels, golden_shape) = load_llama4_pixels(&npz_path); let golden_aspect_ratios = load_llama4_aspect_ratios(&npz_path); let golden_num_tokens = load_golden_num_tokens(&npz_path); // Process image with our Rust processor let image = image::open(&image_path).expect("Failed to open image"); let processor = Llama4VisionProcessor::from_preprocessor_config(&config); let result = processor .preprocess(&[image], &config) .expect("Processing failed"); // Check aspect_ratios let rust_aspect_ratios: Vec<(u32, u32)> = match result.model_specific.get("aspect_ratios") { Some(ModelSpecificValue::UintTensor { data, shape }) => { let num_images = shape[0]; (0..num_images) .map(|i| (data[i * 2], data[i * 2 + 1])) .collect() } _ => panic!("Expected aspect_ratios in model_specific"), }; println!( "llama4_vision - {} image - Aspect ratios: golden={:?}, rust={:?}", image_name, golden_aspect_ratios, rust_aspect_ratios ); assert_eq!( golden_aspect_ratios, rust_aspect_ratios, "aspect_ratios mismatch for {}", image_name ); // Check num_tokens let rust_num_tokens = result.num_img_tokens[0]; println!( "llama4_vision - {} image - Tokens: golden={}, rust={}", image_name, golden_num_tokens, rust_num_tokens ); assert_eq!( golden_num_tokens, rust_num_tokens, "num_tokens mismatch for {}", image_name ); // Check output shape - HuggingFace outputs (num_tiles, 3, 336, 336) without batch // Our Rust outputs (batch, num_tiles, 3, 336, 336) with batch dimension let rust_shape = result.pixel_values.shape(); println!( "llama4_vision - {} image - Shape: golden={:?}, rust={:?}", image_name, golden_shape, rust_shape ); // HuggingFace returns without batch dim, we add batch=1 assert!( rust_shape[0] == 1, "Expected batch dim to be 1, got {}", rust_shape[0] ); assert!( rust_shape[1] >= golden_shape[0], "Expected at least {} tiles, got {}", golden_shape[0], rust_shape[1] ); // Compare pixel values let rust_pixels = result.pixel_values_flat(); let num_golden_elements: usize = golden_shape.iter().product(); // Find the max difference for the actual tiles (not padding) let mut max_diff = 0.0f32; for i in 0..num_golden_elements { let diff = (rust_pixels[i] - golden_pixels[i]).abs(); max_diff = max_diff.max(diff); } println!( "llama4_vision - {} image - Max pixel diff: {:.6}", image_name, max_diff ); // Allow tolerance for floating point and interpolation differences // LLaMA 4 uses bfloat16 internally which may cause small differences assert!( max_diff < 0.03, "Max pixel difference {} exceeds tolerance 0.03 for {}", max_diff, image_name ); } #[test] fn test_llama4_vision_golden_square() { run_llama4_vision_golden_test("square"); } #[test] fn test_llama4_vision_golden_tall() { run_llama4_vision_golden_test("tall"); } #[test] fn test_llama4_vision_golden_wide() { run_llama4_vision_golden_test("wide"); } #[test] fn test_llama4_vision_golden_small() { run_llama4_vision_golden_test("small"); } #[test] fn test_llama4_vision_golden_tiny() { run_llama4_vision_golden_test("tiny"); } #[test] fn test_llama4_vision_golden_very_tall() { run_llama4_vision_golden_test("very_tall"); } #[test] fn test_llama4_vision_golden_very_wide() { run_llama4_vision_golden_test("very_wide"); } #[test] fn test_llama4_vision_golden_large() { run_llama4_vision_golden_test("large"); } #[test] fn test_llama4_vision_golden_odd_dims() { run_llama4_vision_golden_test("odd_dims"); } #[test] fn test_llama4_vision_golden_grayscale() { run_llama4_vision_golden_test("grayscale"); } // ============================================================================ // Pixtral/Mistral3 Vision tests // ============================================================================ /// Load image_sizes from npz file for Pixtral fn load_pixtral_image_sizes(path: &Path) -> Vec<(usize, usize)> { let file = File::open(path).expect("Failed to open golden file"); let mut npz = npyz::npz::NpzArchive::new(file).expect("Failed to parse npz"); let reader = npz .by_name("image_sizes") .expect("Failed to read npz") .expect("No image_sizes"); let shape = reader.shape().to_vec(); // Read data as i64 vec (numpy default for int) let data: Vec = reader.into_vec().expect("Failed to read array"); // Convert to Vec<(usize, usize)> let num_images = shape[0] as usize; (0..num_images) .map(|i| (data[i * 2] as usize, data[i * 2 + 1] as usize)) .collect() } /// Run a Pixtral golden test for a specific image. /// /// This test validates: /// 1. Output shape matches (batch, 3, H, W) /// 2. image_sizes match /// 3. Pixel values match HuggingFace output /// 4. Token count is correct /// /// Pixtral processing: /// - Longest edge: 1024 (default) /// - Patch size: 16 /// - Normalization: CLIP mean/std /// - No tiling - single output per image fn run_pixtral_golden_test(image_name: &str) { let golden_dir = Path::new("tests/fixtures/golden/pixtral"); let image_path = Path::new("tests/fixtures/images").join(format!("{}.jpg", image_name)); if !golden_dir.exists() || !image_path.exists() { eprintln!( "Golden test fixtures for pixtral/{} not found, skipping test", image_name ); eprintln!("Run: python scripts/generate_vision_golden.py --model pixtral"); return; } let npz_path = golden_dir.join(format!("golden_{}.npz", image_name)); let config = load_config(&golden_dir.join("preprocessor_config.json")); // Load golden values let golden_pixels = load_golden_npz(&npz_path); let golden_shape: Vec = golden_pixels.shape().to_vec(); let golden_image_sizes = load_pixtral_image_sizes(&npz_path); let golden_num_tokens = load_golden_num_tokens(&npz_path); // Process image with our Rust processor let image = image::open(&image_path).expect("Failed to open image"); let processor = PixtralProcessor::from_preprocessor_config(&config); let result = processor .preprocess(&[image], &config) .expect("Processing failed"); // Check image_sizes from model_specific let rust_image_sizes: Vec<(usize, usize)> = match result.model_specific.get("image_sizes") { Some(ModelSpecificValue::IntTensor { data, shape }) => { let num_images = shape[0]; (0..num_images) .map(|i| (data[i * 2] as usize, data[i * 2 + 1] as usize)) .collect() } _ => panic!("Expected image_sizes in model_specific"), }; println!( "pixtral - {} image - Image sizes: golden={:?}, rust={:?}", image_name, golden_image_sizes, rust_image_sizes ); assert_eq!( golden_image_sizes, rust_image_sizes, "image_sizes mismatch for {}", image_name ); // Check num_tokens let rust_num_tokens = result.num_img_tokens[0]; println!( "pixtral - {} image - Tokens: golden={}, rust={}", image_name, golden_num_tokens, rust_num_tokens ); assert_eq!( golden_num_tokens, rust_num_tokens, "num_tokens mismatch for {}", image_name ); // Check output shape let rust_shape = result.pixel_values.shape(); println!( "pixtral - {} image - Shape: golden={:?}, rust={:?}", image_name, golden_shape, rust_shape ); // Pixtral outputs [batch, C, H, W] with padding to max size in batch // Single image should match golden shape exactly assert_eq!(rust_shape[0], 1, "Expected batch dim to be 1"); assert_eq!(rust_shape[1], golden_shape[1], "Channel mismatch"); assert!( rust_shape[2] >= golden_shape[2], "Height {} < golden height {}", rust_shape[2], golden_shape[2] ); assert!( rust_shape[3] >= golden_shape[3], "Width {} < golden width {}", rust_shape[3], golden_shape[3] ); // Compare pixel values - only compare the actual image region, not padding let rust_pixels = result.pixel_values_flat(); let golden_pixels_flat: Vec = golden_pixels.iter().copied().collect(); // Calculate indices for the actual image region (not padding) let h = golden_shape[2]; let w = golden_shape[3]; let rust_w = rust_shape[3]; let mut max_diff = 0.0f32; for c in 0..3 { for y in 0..h { for x in 0..w { let golden_idx = c * h * w + y * w + x; let rust_idx = c * rust_shape[2] * rust_w + y * rust_w + x; let diff = (rust_pixels[rust_idx] - golden_pixels_flat[golden_idx]).abs(); max_diff = max_diff.max(diff); } } } println!( "pixtral - {} image - Max pixel diff: {:.6}", image_name, max_diff ); // Allow tolerance for bicubic interpolation differences between PIL and Rust image library // Pixtral uses bicubic which has larger differences than bilinear assert!( max_diff < 0.06, "Max pixel difference {} exceeds tolerance 0.06 for {}", max_diff, image_name ); } #[test] fn test_pixtral_golden_square() { run_pixtral_golden_test("square"); } #[test] fn test_pixtral_golden_tall() { run_pixtral_golden_test("tall"); } #[test] fn test_pixtral_golden_wide() { run_pixtral_golden_test("wide"); } #[test] fn test_pixtral_golden_small() { run_pixtral_golden_test("small"); } #[test] fn test_pixtral_golden_tiny() { run_pixtral_golden_test("tiny"); } #[test] fn test_pixtral_golden_very_tall() { run_pixtral_golden_test("very_tall"); } #[test] fn test_pixtral_golden_very_wide() { run_pixtral_golden_test("very_wide"); } #[test] fn test_pixtral_golden_large() { run_pixtral_golden_test("large"); } #[test] fn test_pixtral_golden_odd_dims() { run_pixtral_golden_test("odd_dims"); } #[test] fn test_pixtral_golden_grayscale() { run_pixtral_golden_test("grayscale"); }