diff --git a/sgl-router/scripts/generate_vision_golden.py b/sgl-router/scripts/generate_vision_golden.py index 4f02beac2..a1ef0f45a 100755 --- a/sgl-router/scripts/generate_vision_golden.py +++ b/sgl-router/scripts/generate_vision_golden.py @@ -41,11 +41,11 @@ MODELS = { "processor_class": "LlavaNextImageProcessor", "description": "Multi-crop anyres processing", }, - # Future models: - # "qwen2_vl": { - # "model_id": "Qwen/Qwen2-VL-7B-Instruct", - # "processor_class": "Qwen2VLImageProcessor", - # }, + "qwen2_vl": { + "model_id": "Qwen/Qwen2-VL-7B-Instruct", + "processor_class": "Qwen2VLImageProcessor", + "description": "Dynamic resolution with smart resize", + }, } # Default test images @@ -189,6 +189,77 @@ def generate_golden_llava_next(image_path: str, output_dir: str) -> dict: return result +def generate_golden_qwen2_vl(image_path: str, output_dir: str) -> dict: + """Generate golden output for Qwen2-VL. + + Qwen2-VL uses dynamic resolution with smart resize: + 1. Smart resize to fit within min/max pixel bounds + 2. Align dimensions to (patch_size * merge_size) boundary + 3. Normalize with CLIP mean/std + 4. Returns image_grid_thw for position encoding + + Default parameters: + - patch_size: 14 + - merge_size: 2 + - min_pixels: 256 * 28 * 28 = 200,704 + - max_pixels: 1280 * 28 * 28 = 1,003,520 + - temporal_patch_size: 2 + """ + try: + from transformers import Qwen2VLImageProcessor + except ImportError: + print("Qwen2VLImageProcessor not available, skipping qwen2_vl") + return None + + processor = Qwen2VLImageProcessor.from_pretrained("Qwen/Qwen2-VL-7B-Instruct") + image = Image.open(image_path).convert("RGB") + original_size = image.size + + # Process image + outputs = processor(images=image, return_tensors="np") + pixel_values = outputs["pixel_values"] + image_grid_thw = outputs.get("image_grid_thw") + + # Get config values for token calculation + patch_size = processor.patch_size + merge_size = processor.merge_size + temporal_patch_size = getattr(processor, "temporal_patch_size", 2) + min_pixels = processor.min_pixels + max_pixels = processor.max_pixels + + # Calculate number of tokens + # tokens = (T * H * W) / merge_size² + if image_grid_thw is not None: + # image_grid_thw has shape [batch, 3] with [T, H, W] + grid_thw = image_grid_thw[0] # First (and only) image + num_tokens = int(np.prod(grid_thw) / (merge_size**2)) + else: + num_tokens = None + + result = { + "pixel_values": pixel_values, + "original_size": original_size, + "processor_config": processor.to_dict(), + } + + if image_grid_thw is not None: + result["image_grid_thw"] = np.array(image_grid_thw) + + if num_tokens is not None: + result["num_tokens"] = num_tokens + + # Add debug info + result["config_info"] = { + "patch_size": patch_size, + "merge_size": merge_size, + "temporal_patch_size": temporal_patch_size, + "min_pixels": min_pixels, + "max_pixels": max_pixels, + } + + return result + + def save_golden(model_key: str, image_name: str, data: dict, output_dir: str): """Save golden output to files.""" model_dir = Path(output_dir) / model_key @@ -220,6 +291,7 @@ def generate_for_model(model_key: str, image_paths: list, output_dir: str): "llava": generate_golden_llava, "llava_pad": generate_golden_llava_pad, "llava_next": generate_golden_llava_next, + "qwen2_vl": generate_golden_qwen2_vl, }.get(model_key) if generator_fn is None: diff --git a/sgl-router/src/multimodal/vision/image_processor.rs b/sgl-router/src/multimodal/vision/image_processor.rs index a996f3522..d8e9e3cfd 100644 --- a/sgl-router/src/multimodal/vision/image_processor.rs +++ b/sgl-router/src/multimodal/vision/image_processor.rs @@ -264,6 +264,7 @@ impl ImageProcessorRegistry { /// Currently registers: /// - `llava-next` -> LlavaNextProcessor /// - `llava` -> LlavaProcessor (also matches llava-1.5, etc.) + /// - `qwen2-vl` -> Qwen2VLProcessor pub fn with_defaults() -> Self { let mut registry = Self::new(); @@ -280,6 +281,16 @@ impl ImageProcessorRegistry { // Register standard LLaVA (matches llava-1.5, llava-v1.5, etc.) registry.register("llava", Box::new(super::processors::LlavaProcessor::new())); + // Register Qwen2-VL (matches Qwen/Qwen2-VL-*, etc.) + registry.register( + "qwen2-vl", + Box::new(super::processors::Qwen2VLProcessor::new()), + ); + registry.register( + "qwen2_vl", + Box::new(super::processors::Qwen2VLProcessor::new()), + ); + registry } } diff --git a/sgl-router/src/multimodal/vision/mod.rs b/sgl-router/src/multimodal/vision/mod.rs index e42cbe066..e59b22cc9 100644 --- a/sgl-router/src/multimodal/vision/mod.rs +++ b/sgl-router/src/multimodal/vision/mod.rs @@ -39,5 +39,5 @@ pub use image_processor::{ ImagePreProcessor, ImageProcessorRegistry, ModelSpecificValue, PreprocessedImages, }; pub use preprocessor_config::PreProcessorConfig; -pub use processors::{LlavaNextProcessor, LlavaProcessor}; +pub use processors::{LlavaNextProcessor, LlavaProcessor, Qwen2VLProcessor}; pub use transforms::TransformError; diff --git a/sgl-router/src/multimodal/vision/processors/mod.rs b/sgl-router/src/multimodal/vision/processors/mod.rs index 378841feb..07e68effa 100644 --- a/sgl-router/src/multimodal/vision/processors/mod.rs +++ b/sgl-router/src/multimodal/vision/processors/mod.rs @@ -7,7 +7,10 @@ //! //! - **LLaVA 1.5** (`llava`): CLIP-based preprocessing with configurable aspect ratio //! - **LLaVA-NeXT** (`llava`): Multi-crop anyres processing +//! - **Qwen2-VL** (`qwen2_vl`): Dynamic resolution with smart resizing pub mod llava; +pub mod qwen2_vl; pub use llava::{ImageAspectRatio, LlavaNextProcessor, LlavaProcessor}; +pub use qwen2_vl::Qwen2VLProcessor; diff --git a/sgl-router/src/multimodal/vision/processors/qwen2_vl.rs b/sgl-router/src/multimodal/vision/processors/qwen2_vl.rs new file mode 100644 index 000000000..893a14901 --- /dev/null +++ b/sgl-router/src/multimodal/vision/processors/qwen2_vl.rs @@ -0,0 +1,682 @@ +//! Qwen2-VL family image processors. +//! +//! This module implements preprocessing for Qwen2-VL models, which use dynamic +//! resolution with smart resizing to maintain aspect ratio within pixel bounds. +//! +//! # Key Features +//! +//! - **Smart Resize**: Resizes images to fit within min/max pixel bounds while +//! preserving aspect ratio and aligning to patch boundaries +//! - **Dynamic Token Count**: Token count depends on actual image dimensions +//! - **image_grid_thw**: Returns (T, H, W) grid dimensions for position encoding +//! +//! # Processing Pipeline +//! +//! 1. Validate aspect ratio (must be < 200:1) +//! 2. Smart resize to fit within min/max pixel bounds +//! 3. Align dimensions to (patch_size * merge_size) boundary +//! 4. Convert to tensor and normalize with CLIP mean/std +//! 5. Reshape into patches for the vision encoder +//! +//! # Token Calculation +//! +//! For Qwen2-VL, the number of image tokens is: +//! ```text +//! grid_t = 1 (for images, temporal dimension is 1) +//! grid_h = resized_height / patch_size +//! grid_w = resized_width / patch_size +//! num_tokens = (grid_t * grid_h * grid_w) / merge_size² +//! ``` + +use image::{DynamicImage, GenericImageView}; +use ndarray::Array3; + +use crate::multimodal::vision::{ + image_processor::{ImagePreProcessor, ModelSpecificValue, PreprocessedImages}, + preprocessor_config::PreProcessorConfig, + transforms::{normalize, pil_to_filter, resize, stack_batch, to_tensor, TransformError}, +}; + +/// CLIP normalization mean values used by Qwen2-VL models. +pub const CLIP_MEAN: [f64; 3] = [0.48145466, 0.4578275, 0.40821073]; + +/// CLIP normalization std values used by Qwen2-VL models. +pub const CLIP_STD: [f64; 3] = [0.26862954, 0.26130258, 0.27577711]; + +/// Default minimum pixels (256 * 28 * 28 = 200,704) +pub const DEFAULT_MIN_PIXELS: usize = 256 * 28 * 28; + +/// Default maximum pixels (1280 * 28 * 28 = 1,003,520) +pub const DEFAULT_MAX_PIXELS: usize = 1280 * 28 * 28; + +/// Default patch size +pub const DEFAULT_PATCH_SIZE: usize = 14; + +/// Default merge size for token reduction +pub const DEFAULT_MERGE_SIZE: usize = 2; + +/// Default temporal patch size (for video frames) +pub const DEFAULT_TEMPORAL_PATCH_SIZE: usize = 2; + +/// Qwen2-VL image processor. +/// +/// Implements dynamic resolution preprocessing with smart resizing that: +/// - Maintains aspect ratio +/// - Fits within configurable min/max pixel bounds +/// - Aligns to patch boundaries for efficient vision encoding +/// +/// +/// The processor returns `image_grid_thw` in the model-specific outputs, +/// which contains the (T, H, W) grid dimensions needed for rotary position +/// encoding in the Qwen2-VL model. +#[derive(Debug, Clone)] +pub struct Qwen2VLProcessor { + /// Vision encoder patch size (typically 14) + pub patch_size: usize, + /// Merge size for token reduction (typically 2) + pub merge_size: usize, + /// Minimum total pixels allowed + pub min_pixels: usize, + /// Maximum total pixels allowed + pub max_pixels: usize, + /// Temporal patch size for video (typically 2) + pub temporal_patch_size: usize, +} + +impl Default for Qwen2VLProcessor { + fn default() -> Self { + Self::new() + } +} + +impl Qwen2VLProcessor { + /// Create a new Qwen2-VL processor with default settings. + /// + /// Defaults: + /// - patch_size: 14 + /// - merge_size: 2 + /// - min_pixels: 200,704 (256 * 28 * 28) + /// - max_pixels: 1,003,520 (1280 * 28 * 28) + /// - temporal_patch_size: 2 + pub fn new() -> Self { + Self { + patch_size: DEFAULT_PATCH_SIZE, + merge_size: DEFAULT_MERGE_SIZE, + min_pixels: DEFAULT_MIN_PIXELS, + max_pixels: DEFAULT_MAX_PIXELS, + temporal_patch_size: DEFAULT_TEMPORAL_PATCH_SIZE, + } + } + + /// Create a processor with custom settings. + pub fn with_config( + patch_size: usize, + merge_size: usize, + min_pixels: usize, + max_pixels: usize, + temporal_patch_size: usize, + ) -> Self { + Self { + patch_size, + merge_size, + min_pixels, + max_pixels, + temporal_patch_size, + } + } + + /// Create a processor from preprocessor config. + pub fn from_preprocessor_config(config: &PreProcessorConfig) -> Self { + Self { + patch_size: config.patch_size.unwrap_or(DEFAULT_PATCH_SIZE), + merge_size: config.merge_size.unwrap_or(DEFAULT_MERGE_SIZE), + min_pixels: config.min_pixels.unwrap_or(DEFAULT_MIN_PIXELS), + max_pixels: config.max_pixels.unwrap_or(DEFAULT_MAX_PIXELS), + temporal_patch_size: config + .temporal_patch_size + .unwrap_or(DEFAULT_TEMPORAL_PATCH_SIZE), + } + } + + /// Get the factor for dimension alignment. + /// + /// Dimensions must be divisible by (patch_size * merge_size). + #[inline] + pub fn get_factor(&self) -> usize { + self.patch_size * self.merge_size + } + + /// Smart resize algorithm for Qwen2-VL. + /// + /// Resizes image dimensions to fit within min/max pixel bounds while: + /// - Preserving aspect ratio + /// - Aligning to (patch_size * merge_size) boundaries + /// + /// # Arguments + /// * `height` - Original image height + /// * `width` - Original image width + /// + /// # Returns + /// (new_height, new_width) or error if aspect ratio is too extreme + /// + /// # Errors + /// - If height or width is smaller than the factor + /// - If aspect ratio exceeds 200:1 + pub fn smart_resize( + &self, + height: usize, + width: usize, + ) -> Result<(usize, usize), TransformError> { + let factor = self.get_factor(); + + // Validate minimum dimensions + if height < factor || width < factor { + return Err(TransformError::InvalidShape { + expected: format!("dimensions >= {} (patch_size * merge_size)", factor), + actual: vec![height, width], + }); + } + + // Validate aspect ratio + let max_dim = height.max(width) as f64; + let min_dim = height.min(width) as f64; + let aspect_ratio = max_dim / min_dim; + if aspect_ratio > 200.0 { + return Err(TransformError::InvalidShape { + expected: "aspect ratio < 200:1".to_string(), + actual: vec![height, width], + }); + } + + // Round to nearest factor multiple + let mut h_bar = (height as f64 / factor as f64).round() as usize * factor; + let mut w_bar = (width as f64 / factor as f64).round() as usize * factor; + + // Ensure minimum size + h_bar = h_bar.max(factor); + w_bar = w_bar.max(factor); + + // Scale down if exceeding max_pixels + if h_bar * w_bar > self.max_pixels { + let beta = ((height * width) as f64 / self.max_pixels as f64).sqrt(); + h_bar = ((height as f64 / beta / factor as f64).floor() as usize) * factor; + w_bar = ((width as f64 / beta / factor as f64).floor() as usize) * factor; + // Ensure minimum size after scaling down + h_bar = h_bar.max(factor); + w_bar = w_bar.max(factor); + } + // Scale up if below min_pixels + else if h_bar * w_bar < self.min_pixels { + let beta = (self.min_pixels as f64 / (height * width) as f64).sqrt(); + h_bar = ((height as f64 * beta / factor as f64).ceil() as usize) * factor; + w_bar = ((width as f64 * beta / factor as f64).ceil() as usize) * factor; + } + + Ok((h_bar, w_bar)) + } + + /// Calculate the grid dimensions (T, H, W) for an image. + /// + /// For single images, T=1. For video, T = num_frames / temporal_patch_size. + /// + /// # Arguments + /// * `height` - Resized image height + /// * `width` - Resized image width + /// * `num_frames` - Number of frames (1 for images) + /// + /// # Returns + /// (grid_t, grid_h, grid_w) + pub fn calculate_grid_thw( + &self, + height: usize, + width: usize, + num_frames: usize, + ) -> (usize, usize, usize) { + let grid_t = num_frames.max(self.temporal_patch_size) / self.temporal_patch_size; + let grid_h = height / self.patch_size; + let grid_w = width / self.patch_size; + (grid_t, grid_h, grid_w) + } + + /// Calculate the number of image tokens after merge. + /// + /// tokens = (grid_t * grid_h * grid_w) / merge_size² + pub fn calculate_tokens_from_grid(&self, grid_t: usize, grid_h: usize, grid_w: usize) -> usize { + (grid_t * grid_h * grid_w) / (self.merge_size * self.merge_size) + } + + /// Reshape pixel values from [C, H, W] to flattened patches format. + /// + /// This matches the HuggingFace Qwen2VLImageProcessor output format: + /// `(num_patches, patch_features)` where: + /// - num_patches = grid_t * grid_h * grid_w + /// - patch_features = C * temporal_patch_size * patch_size * patch_size + /// + /// The transformation follows these steps (matching HuggingFace exactly): + /// 1. Start with [C, H, W] tensor, expand to [temporal, C, H, W] + /// 2. Reshape to [grid_t, temporal, C, grid_h/merge, merge, patch, grid_w/merge, merge, patch] + /// 3. Permute to [grid_t, grid_h/merge, grid_w/merge, merge, merge, C, temporal, patch, patch] + /// 4. Flatten to [num_patches, patch_features] + /// + /// # Arguments + /// * `tensor` - Input tensor of shape [C, H, W] + /// * `grid_t` - Temporal grid size (1 for images) + /// * `grid_h` - Height grid size (H / patch_size) + /// * `grid_w` - Width grid size (W / patch_size) + /// + /// # Returns + /// Flattened patches as Vec with shape semantics (num_patches, patch_features) + pub fn reshape_to_patches( + &self, + tensor: &Array3, + grid_t: usize, + grid_h: usize, + grid_w: usize, + ) -> Vec { + use ndarray::IxDyn; + + let channel = tensor.shape()[0]; + let height = tensor.shape()[1]; + let width = tensor.shape()[2]; + + let patch_size = self.patch_size; + let merge_size = self.merge_size; + let temporal_patch_size = self.temporal_patch_size; + + // Verify dimensions match expected grid + debug_assert_eq!( + height, + grid_h * patch_size, + "Height must match grid_h * patch_size" + ); + debug_assert_eq!( + width, + grid_w * patch_size, + "Width must match grid_w * patch_size" + ); + + // Step 1: Expand temporal dimension by replicating the frame + // [C, H, W] -> [temporal_patch_size, C, H, W] + let expanded = tensor + .view() + .insert_axis(ndarray::Axis(0)) + .broadcast((temporal_patch_size, channel, height, width)) + .expect("Broadcast failed") + .to_owned(); + + // Step 2: Reshape to split spatial dimensions into grid and patch components + // [temporal, C, H, W] -> [grid_t, temporal, C, grid_h/merge, merge, patch, grid_w/merge, merge, patch] + // + // Note: For images, grid_t=1 and we have temporal_patch_size frames (replicated) + // HF reshape: [grid_t, temporal, C, grid_h/merge, merge, patch, grid_w/merge, merge, patch] + let grid_h_merged = grid_h / merge_size; + let grid_w_merged = grid_w / merge_size; + + // Use IxDyn for 9-dimensional reshape (ndarray only supports up to Ix6 for fixed dims) + let shape_9d = IxDyn(&[ + grid_t, + temporal_patch_size, + channel, + grid_h_merged, + merge_size, + patch_size, + grid_w_merged, + merge_size, + patch_size, + ]); + + let reshaped = expanded + .into_shape_with_order(shape_9d) + .expect("Reshape failed"); + + // Step 3: Permute axes to match HuggingFace output order + // From: [grid_t, temporal, C, grid_h/merge, merge, patch, grid_w/merge, merge, patch] + // [ 0 , 1 , 2, 3 , 4 , 5 , 6 , 7 , 8 ] + // To: [grid_t, grid_h/merge, grid_w/merge, merge, merge, C, temporal, patch, patch] + // [ 0 , 3 , 6 , 4 , 7 , 2, 1 , 5 , 8 ] + let permuted = reshaped.permuted_axes(&[0, 3, 6, 4, 7, 2, 1, 5, 8][..]); + + // Step 4: Flatten to [num_patches, patch_features] + // num_patches = grid_t * grid_h * grid_w = grid_t * (grid_h/merge * merge) * (grid_w/merge * merge) + // patch_features = C * temporal * patch * patch + let num_patches = grid_t * grid_h * grid_w; + let patch_features = channel * temporal_patch_size * patch_size * patch_size; + + // Make contiguous and flatten + let contiguous = permuted.as_standard_layout().into_owned(); + let flat = contiguous + .into_shape_with_order(IxDyn(&[num_patches, patch_features])) + .expect("Final reshape failed"); + + let (vec, _offset) = flat.into_raw_vec_and_offset(); + vec + } +} + +impl ImagePreProcessor for Qwen2VLProcessor { + fn default_mean(&self) -> [f64; 3] { + CLIP_MEAN + } + + fn default_std(&self) -> [f64; 3] { + CLIP_STD + } + + fn preprocess( + &self, + images: &[DynamicImage], + config: &PreProcessorConfig, + ) -> Result { + if images.is_empty() { + return Err(TransformError::EmptyBatch); + } + + // Store original sizes + let image_sizes: Vec<(u32, u32)> = images.iter().map(|img| img.dimensions()).collect(); + + // For Qwen2-VL, when batching multiple images, they may have different + // resized dimensions. We need to either: + // 1. Process each image separately and return individual tensors + // 2. Pad all images to the max dimensions in the batch + // + // Following mistral.rs approach: find max dimensions and resize all to that + + // First pass: calculate target dimensions for each image + let mut target_sizes = Vec::with_capacity(images.len()); + for image in images { + let (w, h) = image.dimensions(); + let (new_h, new_w) = self.smart_resize(h as usize, w as usize)?; + target_sizes.push((new_h, new_w)); + } + + // Find max height and width across all images + let max_height = target_sizes.iter().map(|(h, _)| *h).max().unwrap_or(0); + let max_width = target_sizes.iter().map(|(_, w)| *w).max().unwrap_or(0); + + // Process each image with uniform max dimensions + let mean = config.get_image_mean(); + let std = config.get_image_std(); + let filter = pil_to_filter(config.resampling); + + let mut tensors = Vec::with_capacity(images.len()); + let mut grid_thw_data = Vec::with_capacity(images.len() * 3); + let mut num_img_tokens = Vec::with_capacity(images.len()); + + for (i, image) in images.iter().enumerate() { + let (target_h, target_w) = target_sizes[i]; + + // Resize to the target size for this image + let resized = if config.do_resize.unwrap_or(true) { + // For batching: resize to max dimensions to enable stacking + // The actual grid dimensions are based on individual target sizes + resize(image, max_width as u32, max_height as u32, filter) + } else { + image.clone() + }; + + // Convert to tensor + let mut tensor = to_tensor(&resized); + + // Normalize + if config.do_normalize.unwrap_or(true) { + normalize(&mut tensor, &mean, &std); + } + + tensors.push(tensor); + + // Grid dimensions are based on the individual image's target size + let (grid_t, grid_h, grid_w) = self.calculate_grid_thw(target_h, target_w, 1); + grid_thw_data.push(grid_t as u32); + grid_thw_data.push(grid_h as u32); + grid_thw_data.push(grid_w as u32); + + // Token count is based on individual grid + let tokens = self.calculate_tokens_from_grid(grid_t, grid_h, grid_w); + num_img_tokens.push(tokens); + } + + // Stack tensors into batch (now all same size) + let pixel_values = stack_batch(&tensors)?; + + // Create result with model-specific image_grid_thw + let result = PreprocessedImages::new(pixel_values, num_img_tokens, image_sizes).with_extra( + "image_grid_thw", + ModelSpecificValue::uint_2d(grid_thw_data, images.len(), 3), + ); + + Ok(result) + } + + fn calculate_num_tokens(&self, width: u32, height: u32, _config: &PreProcessorConfig) -> usize { + // Calculate resized dimensions + let (new_height, new_width) = match self.smart_resize(height as usize, width as usize) { + Ok((h, w)) => (h, w), + Err(_) => { + // Fallback: use minimum size + let factor = self.get_factor(); + (factor, factor) + } + }; + + // Calculate grid and tokens + let (grid_t, grid_h, grid_w) = self.calculate_grid_thw(new_height, new_width, 1); + self.calculate_tokens_from_grid(grid_t, grid_h, grid_w) + } + + fn model_name(&self) -> &'static str { + "qwen2-vl" + } + + fn get_processed_size(&self, _config: &PreProcessorConfig) -> Option<(u32, u32)> { + // Qwen2-VL has dynamic sizing, no fixed output size + None + } +} + +#[cfg(test)] +mod tests { + use image::{Rgb, RgbImage}; + + use super::*; + + fn create_test_image(width: u32, height: u32, color: Rgb) -> DynamicImage { + DynamicImage::from(RgbImage::from_pixel(width, height, color)) + } + + #[test] + fn test_qwen2_vl_processor_default() { + let processor = Qwen2VLProcessor::new(); + assert_eq!(processor.patch_size, 14); + assert_eq!(processor.merge_size, 2); + assert_eq!(processor.min_pixels, DEFAULT_MIN_PIXELS); + assert_eq!(processor.max_pixels, DEFAULT_MAX_PIXELS); + assert_eq!(processor.get_factor(), 28); // 14 * 2 + } + + #[test] + fn test_smart_resize_within_bounds() { + let processor = Qwen2VLProcessor::new(); + + // Image that's already within bounds + let (h, w) = processor.smart_resize(500, 500).unwrap(); + + // Should be aligned to factor (28) + assert_eq!(h % 28, 0); + assert_eq!(w % 28, 0); + + // Should be within bounds + assert!(h * w >= processor.min_pixels); + assert!(h * w <= processor.max_pixels); + } + + #[test] + fn test_smart_resize_too_large() { + let processor = Qwen2VLProcessor::new(); + + // Very large image + let (h, w) = processor.smart_resize(3000, 3000).unwrap(); + + // Should be scaled down + assert!(h * w <= processor.max_pixels); + assert_eq!(h % 28, 0); + assert_eq!(w % 28, 0); + } + + #[test] + fn test_smart_resize_too_small() { + let processor = Qwen2VLProcessor::new(); + + // Small image (but above minimum dimension) + let (h, w) = processor.smart_resize(100, 100).unwrap(); + + // Should be scaled up to min_pixels + assert!(h * w >= processor.min_pixels); + assert_eq!(h % 28, 0); + assert_eq!(w % 28, 0); + } + + #[test] + fn test_smart_resize_aspect_ratio_preserved() { + let processor = Qwen2VLProcessor::new(); + + // 2:1 aspect ratio + let (h, w) = processor.smart_resize(400, 800).unwrap(); + + // Aspect ratio should be approximately preserved + let original_ratio = 800.0 / 400.0; + let new_ratio = w as f64 / h as f64; + assert!((new_ratio - original_ratio).abs() < 0.5); + } + + #[test] + fn test_smart_resize_extreme_aspect_ratio_error() { + let processor = Qwen2VLProcessor::new(); + + // 300:1 aspect ratio - should fail + let result = processor.smart_resize(100, 30000); + assert!(result.is_err()); + } + + #[test] + fn test_smart_resize_too_small_dimension_error() { + let processor = Qwen2VLProcessor::new(); + + // Dimension smaller than factor + let result = processor.smart_resize(10, 100); + assert!(result.is_err()); + } + + #[test] + fn test_calculate_grid_thw_image() { + let processor = Qwen2VLProcessor::new(); + + // 448x448 image (16x16 grid patches) + let (t, h, w) = processor.calculate_grid_thw(448, 448, 1); + + assert_eq!(t, 1); // Single image + assert_eq!(h, 448 / 14); // 32 + assert_eq!(w, 448 / 14); // 32 + } + + #[test] + fn test_calculate_tokens() { + let processor = Qwen2VLProcessor::new(); + + // With merge_size=2, tokens = (t * h * w) / 4 + let tokens = processor.calculate_tokens_from_grid(1, 32, 32); + assert_eq!(tokens, (32 * 32) / 4); // 256 + } + + #[test] + fn test_qwen2_vl_preprocess() { + let processor = Qwen2VLProcessor::new(); + let config = PreProcessorConfig { + do_resize: Some(true), + do_normalize: Some(true), + image_mean: Some(CLIP_MEAN.to_vec()), + image_std: Some(CLIP_STD.to_vec()), + patch_size: Some(14), + merge_size: Some(2), + min_pixels: Some(DEFAULT_MIN_PIXELS), + max_pixels: Some(DEFAULT_MAX_PIXELS), + ..Default::default() + }; + + let image = create_test_image(600, 400, Rgb([128, 128, 128])); + let result = processor.preprocess(&[image], &config).unwrap(); + + assert_eq!(result.batch_size(), 1); + + // Check pixel values are normalized + let flat = result.pixel_values_flat(); + // After normalization with CLIP mean/std, gray (0.5) should be near 0 + // (0.5 - 0.48) / 0.27 ≈ 0.07 + assert!(flat.iter().all(|&v| v.abs() < 1.0)); // Should be normalized + + // Check image_grid_thw is present + assert!(result.model_specific.contains_key("image_grid_thw")); + + // Verify token count is reasonable + assert!(result.num_img_tokens[0] > 0); + } + + #[test] + fn test_qwen2_vl_preprocess_multiple() { + let processor = Qwen2VLProcessor::new(); + let config = PreProcessorConfig::default(); + + let images = vec![ + create_test_image(600, 400, Rgb([100, 100, 100])), + create_test_image(400, 600, Rgb([150, 150, 150])), + ]; + + let result = processor.preprocess(&images, &config).unwrap(); + + // Both images processed + assert_eq!(result.image_sizes.len(), 2); + assert_eq!(result.num_img_tokens.len(), 2); + + // Check grid_thw shape + if let Some(ModelSpecificValue::UintTensor { data, shape }) = + result.model_specific.get("image_grid_thw") + { + assert_eq!(shape, &[2, 3]); // 2 images, 3 values (T, H, W) each + assert_eq!(data.len(), 6); + } else { + panic!("Expected image_grid_thw to be UintTensor"); + } + } + + #[test] + fn test_qwen2_vl_from_config() { + let config = PreProcessorConfig { + patch_size: Some(16), + merge_size: Some(4), + min_pixels: Some(100000), + max_pixels: Some(500000), + temporal_patch_size: Some(4), + ..Default::default() + }; + + let processor = Qwen2VLProcessor::from_preprocessor_config(&config); + + assert_eq!(processor.patch_size, 16); + assert_eq!(processor.merge_size, 4); + assert_eq!(processor.min_pixels, 100000); + assert_eq!(processor.max_pixels, 500000); + assert_eq!(processor.temporal_patch_size, 4); + } + + #[test] + fn test_model_name() { + let processor = Qwen2VLProcessor::new(); + assert_eq!(processor.model_name(), "qwen2-vl"); + } + + #[test] + fn test_default_mean_std() { + let processor = Qwen2VLProcessor::new(); + assert_eq!(processor.default_mean(), CLIP_MEAN); + assert_eq!(processor.default_std(), CLIP_STD); + } +} diff --git a/sgl-router/tests/vision_golden_tests.rs b/sgl-router/tests/vision_golden_tests.rs index 39b2651f6..cefc91a60 100644 --- a/sgl-router/tests/vision_golden_tests.rs +++ b/sgl-router/tests/vision_golden_tests.rs @@ -3,9 +3,10 @@ //! These tests compare Rust preprocessor output against golden outputs //! generated by HuggingFace transformers to ensure pixel-perfect compatibility. //! -//! Two modes are tested: +//! Modes tested: //! - `llava/` - Standard CLIP processing (llava-hf/* models, no expand-to-square) //! - `llava_pad/` - Expand-to-square mode (liuhaotian/llava-* models, image_aspect_ratio=pad) +//! - `qwen2_vl/` - Dynamic resolution with smart resize (Qwen/Qwen2-VL-* models) //! //! To regenerate golden outputs: //! ```bash @@ -16,7 +17,8 @@ use std::{fs::File, io::Read, path::Path}; use ndarray::Array4; use sgl_model_gateway::multimodal::vision::{ - ImagePreProcessor, LlavaProcessor, PreProcessorConfig, + image_processor::ModelSpecificValue, ImagePreProcessor, LlavaProcessor, PreProcessorConfig, + Qwen2VLProcessor, }; /// Load a numpy .npz file and extract pixel_values @@ -65,6 +67,41 @@ fn max_diff(a: &Array4, b: &Array4) -> f32 { (a - b).mapv(|v| v.abs()).fold(0.0f32, |acc, &v| acc.max(v)) } +/// Load image_grid_thw from npz file +fn load_golden_grid_thw(path: &Path) -> Vec { + let file = File::open(path).expect("Failed to open golden file"); + let mut npz = npyz::npz::NpzArchive::new(file).expect("Failed to parse npz"); + + let reader = npz + .by_name("image_grid_thw") + .expect("Failed to read npz") + .expect("No image_grid_thw"); + + // Shape not needed, data is flat + let _shape = reader.shape(); + + // Read data as i64 vec (numpy default for int) + let data: Vec = reader.into_vec().expect("Failed to read array"); + + // Convert to u32 + data.into_iter().map(|v| v as u32).collect() +} + +/// Load num_tokens from npz file +fn load_golden_num_tokens(path: &Path) -> usize { + let file = File::open(path).expect("Failed to open golden file"); + let mut npz = npyz::npz::NpzArchive::new(file).expect("Failed to parse npz"); + + let reader = npz + .by_name("num_tokens") + .expect("Failed to read npz") + .expect("No num_tokens"); + + // Read single value as i64 + let data: Vec = reader.into_vec().expect("Failed to read array"); + data[0] as usize +} + /// Run a golden test for a specific mode and image. /// /// # Arguments @@ -185,3 +222,165 @@ fn test_llava_token_count() { "Expected 576 tokens for 336x336 with patch_size=14" ); } + +// ============================================================================ +// Qwen2-VL tests +// ============================================================================ + +/// Load flattened pixel values from Qwen2-VL npz file. +/// Returns (data, shape) where shape is (num_patches, patch_features). +fn load_golden_qwen2_vl_pixels(path: &Path) -> (Vec, (usize, usize)) { + let file = File::open(path).expect("Failed to open golden file"); + let mut npz = npyz::npz::NpzArchive::new(file).expect("Failed to parse npz"); + + let reader = npz + .by_name("pixel_values") + .expect("Failed to read npz") + .expect("No pixel_values"); + + let shape = reader.shape().to_vec(); + assert_eq!(shape.len(), 2, "Expected 2D tensor for Qwen2-VL patches"); + + let data: Vec = reader.into_vec().expect("Failed to read array"); + (data, (shape[0] as usize, shape[1] as usize)) +} + +/// Run a Qwen2-VL golden test for a specific image. +/// +/// This test validates: +/// 1. image_grid_thw matches the HuggingFace output +/// 2. num_tokens calculation is correct +/// 3. Pixel values match after reshaping to patch format +fn run_qwen2_vl_golden_test(image_name: &str) { + let golden_dir = Path::new("tests/fixtures/golden/qwen2_vl"); + let image_path = Path::new("tests/fixtures/images").join(format!("{}.jpg", image_name)); + + if !golden_dir.exists() || !image_path.exists() { + eprintln!( + "Golden test fixtures for qwen2_vl/{} not found, skipping test", + image_name + ); + eprintln!("Run: python scripts/generate_vision_golden.py --model qwen2_vl"); + return; + } + + let npz_path = golden_dir.join(format!("golden_{}.npz", image_name)); + let config = load_config(&golden_dir.join("preprocessor_config.json")); + + // Load golden values + let golden_grid_thw = load_golden_grid_thw(&npz_path); + let golden_num_tokens = load_golden_num_tokens(&npz_path); + let (golden_pixels, golden_shape) = load_golden_qwen2_vl_pixels(&npz_path); + + // Process image with our Rust processor + let image = image::open(&image_path).expect("Failed to open image"); + let processor = Qwen2VLProcessor::from_preprocessor_config(&config); + let result = processor + .preprocess(&[image], &config) + .expect("Processing failed"); + + // Extract image_grid_thw from result + let rust_grid_thw = match result.model_specific.get("image_grid_thw") { + Some(ModelSpecificValue::UintTensor { data, shape }) => { + assert_eq!(shape, &[1, 3], "Expected shape [1, 3] for single image"); + data.clone() + } + _ => panic!("Expected image_grid_thw in model_specific"), + }; + + // Compare grid dimensions + println!( + "qwen2_vl - {} image - Grid T H W: golden={:?}, rust={:?}", + image_name, golden_grid_thw, rust_grid_thw + ); + assert_eq!( + golden_grid_thw, rust_grid_thw, + "image_grid_thw mismatch for {}", + image_name + ); + + // Compare token counts + let rust_num_tokens = result.num_img_tokens[0]; + println!( + "qwen2_vl - {} image - Tokens: golden={}, rust={}", + image_name, golden_num_tokens, rust_num_tokens + ); + assert_eq!( + golden_num_tokens, rust_num_tokens, + "num_tokens mismatch for {}", + image_name + ); + + // Compare pixel values by reshaping our output to patch format + let grid_t = rust_grid_thw[0] as usize; + let grid_h = rust_grid_thw[1] as usize; + let grid_w = rust_grid_thw[2] as usize; + + // Get the tensor for the first image (batch index 0) + let pixel_values = &result.pixel_values; + let tensor_3d = pixel_values.index_axis(ndarray::Axis(0), 0).to_owned(); + + // Reshape to patches format + let rust_patches = processor.reshape_to_patches(&tensor_3d, grid_t, grid_h, grid_w); + + // Verify shapes match + let expected_num_patches = grid_t * grid_h * grid_w; + let patch_size = config.patch_size.unwrap_or(14); + let temporal_patch_size = config.temporal_patch_size.unwrap_or(2); + let expected_patch_features = 3 * temporal_patch_size * patch_size * patch_size; + + println!( + "qwen2_vl - {} image - Patch shape: golden={:?}, rust=({}, {})", + image_name, golden_shape, expected_num_patches, expected_patch_features + ); + assert_eq!( + golden_shape, + (expected_num_patches, expected_patch_features), + "Patch shape mismatch" + ); + assert_eq!( + rust_patches.len(), + expected_num_patches * expected_patch_features, + "Rust patches size mismatch" + ); + + // Compare pixel values + let max_diff = rust_patches + .iter() + .zip(golden_pixels.iter()) + .map(|(r, g)| (r - g).abs()) + .fold(0.0f32, f32::max); + + println!( + "qwen2_vl - {} image - Max pixel diff: {:.6}", + image_name, max_diff + ); + + // Allow tolerance for floating point and interpolation differences + assert!( + max_diff < 0.02, + "Max pixel difference {} exceeds tolerance 0.02 for {}", + max_diff, + image_name + ); +} + +#[test] +fn test_qwen2_vl_golden_square() { + run_qwen2_vl_golden_test("square"); +} + +#[test] +fn test_qwen2_vl_golden_tall() { + run_qwen2_vl_golden_test("tall"); +} + +#[test] +fn test_qwen2_vl_golden_wide() { + run_qwen2_vl_golden_test("wide"); +} + +#[test] +fn test_qwen2_vl_golden_small() { + run_qwen2_vl_golden_test("small"); +}