diff --git a/sgl-router/scripts/generate_vision_golden.py b/sgl-router/scripts/generate_vision_golden.py index a1ef0f45a..4209afd89 100755 --- a/sgl-router/scripts/generate_vision_golden.py +++ b/sgl-router/scripts/generate_vision_golden.py @@ -46,6 +46,11 @@ MODELS = { "processor_class": "Qwen2VLImageProcessor", "description": "Dynamic resolution with smart resize", }, + "qwen3_vl": { + "model_id": "Qwen/Qwen3-VL-8B-Instruct", + "processor_class": "Qwen2VLImageProcessorFast", + "description": "Dynamic resolution with patch_size=16 and [0.5,0.5,0.5] normalization", + }, } # Default test images @@ -283,6 +288,72 @@ def save_golden(model_key: str, image_name: str, data: dict, output_dir: str): print(f" Saved: {config_path}") +def generate_golden_qwen3_vl(image_path: str, output_dir: str) -> dict: + """Generate golden output for Qwen3-VL. + + Qwen3-VL uses dynamic resolution with smart resize similar to Qwen2-VL + but with different parameters: + - patch_size: 16 (vs 14 in Qwen2-VL) + - factor: 32 (vs 28 in Qwen2-VL) + - normalization: [0.5, 0.5, 0.5] mean/std (vs CLIP values in Qwen2-VL) + + Default parameters: + - patch_size: 16 + - merge_size: 2 + - temporal_patch_size: 2 + """ + from transformers import AutoProcessor + + processor = AutoProcessor.from_pretrained( + "Qwen/Qwen3-VL-8B-Instruct", trust_remote_code=True + ) + image = Image.open(image_path).convert("RGB") + original_size = image.size + + # Process image using the image processor directly + outputs = processor.image_processor(images=image, return_tensors="pt") + + # Convert to numpy for saving + pixel_values = outputs["pixel_values"].numpy() + image_grid_thw = outputs.get("image_grid_thw") + if image_grid_thw is not None: + image_grid_thw = image_grid_thw.numpy() + + # Get config values + img_processor = processor.image_processor + patch_size = getattr(img_processor, "patch_size", 16) + merge_size = getattr(img_processor, "merge_size", 2) + temporal_patch_size = getattr(img_processor, "temporal_patch_size", 2) + + # Calculate number of tokens + if image_grid_thw is not None: + grid_thw = image_grid_thw[0] + num_tokens = int(np.prod(grid_thw) / (merge_size**2)) + else: + num_tokens = None + + result = { + "pixel_values": pixel_values, + "original_size": original_size, + "processor_config": img_processor.to_dict(), + } + + if image_grid_thw is not None: + result["image_grid_thw"] = image_grid_thw + + if num_tokens is not None: + result["num_tokens"] = num_tokens + + # Add debug info + result["config_info"] = { + "patch_size": patch_size, + "merge_size": merge_size, + "temporal_patch_size": temporal_patch_size, + } + + return result + + def generate_for_model(model_key: str, image_paths: list, output_dir: str): """Generate golden outputs for a specific model.""" print(f"\nGenerating golden outputs for {model_key}...") @@ -292,6 +363,7 @@ def generate_for_model(model_key: str, image_paths: list, output_dir: str): "llava_pad": generate_golden_llava_pad, "llava_next": generate_golden_llava_next, "qwen2_vl": generate_golden_qwen2_vl, + "qwen3_vl": generate_golden_qwen3_vl, }.get(model_key) if generator_fn is None: diff --git a/sgl-router/src/multimodal/vision/image_processor.rs b/sgl-router/src/multimodal/vision/image_processor.rs index 5c305e322..e81527dcc 100644 --- a/sgl-router/src/multimodal/vision/image_processor.rs +++ b/sgl-router/src/multimodal/vision/image_processor.rs @@ -266,6 +266,7 @@ impl ImageProcessorRegistry { /// - `llava` -> LlavaProcessor (also matches llava-1.5, etc.) /// - `qwen2-vl` -> Qwen2VLProcessor /// - `qwen2.5-vl` -> Qwen2VLProcessor (same preprocessing as Qwen2-VL) + /// - `qwen3-vl` -> Qwen3VLProcessor (patch_size=16, [0.5,0.5,0.5] normalization) pub fn with_defaults() -> Self { let mut registry = Self::new(); @@ -282,6 +283,16 @@ impl ImageProcessorRegistry { // Register standard LLaVA (matches llava-1.5, llava-v1.5, etc.) registry.register("llava", Box::new(super::processors::LlavaProcessor::new())); + // Register Qwen3-VL first (more specific pattern - must match before qwen2) + registry.register( + "qwen3-vl", + Box::new(super::processors::Qwen3VLProcessor::new()), + ); + registry.register( + "qwen3_vl", + Box::new(super::processors::Qwen3VLProcessor::new()), + ); + // Register Qwen2-VL (matches Qwen/Qwen2-VL-*, etc.) registry.register( "qwen2-vl", diff --git a/sgl-router/src/multimodal/vision/mod.rs b/sgl-router/src/multimodal/vision/mod.rs index e59b22cc9..a1f896344 100644 --- a/sgl-router/src/multimodal/vision/mod.rs +++ b/sgl-router/src/multimodal/vision/mod.rs @@ -39,5 +39,5 @@ pub use image_processor::{ ImagePreProcessor, ImageProcessorRegistry, ModelSpecificValue, PreprocessedImages, }; pub use preprocessor_config::PreProcessorConfig; -pub use processors::{LlavaNextProcessor, LlavaProcessor, Qwen2VLProcessor}; +pub use processors::{LlavaNextProcessor, LlavaProcessor, Qwen2VLProcessor, Qwen3VLProcessor}; pub use transforms::TransformError; diff --git a/sgl-router/src/multimodal/vision/processors/mod.rs b/sgl-router/src/multimodal/vision/processors/mod.rs index f9a165454..dc3ed82c9 100644 --- a/sgl-router/src/multimodal/vision/processors/mod.rs +++ b/sgl-router/src/multimodal/vision/processors/mod.rs @@ -9,9 +9,13 @@ //! - **LLaVA-NeXT** (`llava`): Multi-crop anyres processing //! - **Qwen2-VL** (`qwen2_vl`): Dynamic resolution with smart resizing //! - **Qwen2.5-VL** (`qwen2_vl`): Same processor as Qwen2-VL (identical preprocessing) +//! - **Qwen3-VL** (`qwen3_vl`): Similar to Qwen2-VL but with patch_size=16 and [0.5,0.5,0.5] normalization pub mod llava; pub mod qwen2_vl; +pub mod qwen3_vl; +pub mod qwen_vl_base; pub use llava::{ImageAspectRatio, LlavaNextProcessor, LlavaProcessor}; pub use qwen2_vl::Qwen2VLProcessor; +pub use qwen3_vl::Qwen3VLProcessor; diff --git a/sgl-router/src/multimodal/vision/processors/qwen2_vl.rs b/sgl-router/src/multimodal/vision/processors/qwen2_vl.rs index 893a14901..614dc2c4c 100644 --- a/sgl-router/src/multimodal/vision/processors/qwen2_vl.rs +++ b/sgl-router/src/multimodal/vision/processors/qwen2_vl.rs @@ -1,7 +1,7 @@ //! Qwen2-VL family image processors. //! -//! This module implements preprocessing for Qwen2-VL models, which use dynamic -//! resolution with smart resizing to maintain aspect ratio within pixel bounds. +//! This module provides the Qwen2-VL processor which wraps the shared +//! `QwenVLProcessorBase` with Qwen2-VL specific default parameters. //! //! # Key Features //! @@ -10,31 +10,23 @@ //! - **Dynamic Token Count**: Token count depends on actual image dimensions //! - **image_grid_thw**: Returns (T, H, W) grid dimensions for position encoding //! -//! # Processing Pipeline +//! # Qwen2-VL Parameters //! -//! 1. Validate aspect ratio (must be < 200:1) -//! 2. Smart resize to fit within min/max pixel bounds -//! 3. Align dimensions to (patch_size * merge_size) boundary -//! 4. Convert to tensor and normalize with CLIP mean/std -//! 5. Reshape into patches for the vision encoder -//! -//! # Token Calculation -//! -//! For Qwen2-VL, the number of image tokens is: -//! ```text -//! grid_t = 1 (for images, temporal dimension is 1) -//! grid_h = resized_height / patch_size -//! grid_w = resized_width / patch_size -//! num_tokens = (grid_t * grid_h * grid_w) / merge_size² -//! ``` +//! - patch_size: 14 +//! - merge_size: 2 +//! - factor: 28 (patch_size * merge_size) +//! - normalization: CLIP mean/std -use image::{DynamicImage, GenericImageView}; +use std::ops::Deref; + +use image::DynamicImage; use ndarray::Array3; +use super::qwen_vl_base::{QwenVLConfig, QwenVLProcessorBase}; use crate::multimodal::vision::{ - image_processor::{ImagePreProcessor, ModelSpecificValue, PreprocessedImages}, + image_processor::{ImagePreProcessor, PreprocessedImages}, preprocessor_config::PreProcessorConfig, - transforms::{normalize, pil_to_filter, resize, stack_batch, to_tensor, TransformError}, + transforms::TransformError, }; /// CLIP normalization mean values used by Qwen2-VL models. @@ -60,27 +52,14 @@ pub const DEFAULT_TEMPORAL_PATCH_SIZE: usize = 2; /// Qwen2-VL image processor. /// -/// Implements dynamic resolution preprocessing with smart resizing that: -/// - Maintains aspect ratio -/// - Fits within configurable min/max pixel bounds -/// - Aligns to patch boundaries for efficient vision encoding -/// -/// -/// The processor returns `image_grid_thw` in the model-specific outputs, -/// which contains the (T, H, W) grid dimensions needed for rotary position -/// encoding in the Qwen2-VL model. +/// This is a thin wrapper around `QwenVLProcessorBase` with Qwen2-VL +/// specific default parameters: +/// - patch_size: 14 +/// - merge_size: 2 +/// - CLIP normalization mean/std #[derive(Debug, Clone)] pub struct Qwen2VLProcessor { - /// Vision encoder patch size (typically 14) - pub patch_size: usize, - /// Merge size for token reduction (typically 2) - pub merge_size: usize, - /// Minimum total pixels allowed - pub min_pixels: usize, - /// Maximum total pixels allowed - pub max_pixels: usize, - /// Temporal patch size for video (typically 2) - pub temporal_patch_size: usize, + inner: QwenVLProcessorBase, } impl Default for Qwen2VLProcessor { @@ -98,13 +77,19 @@ impl Qwen2VLProcessor { /// - min_pixels: 200,704 (256 * 28 * 28) /// - max_pixels: 1,003,520 (1280 * 28 * 28) /// - temporal_patch_size: 2 + /// - normalization: CLIP mean/std pub fn new() -> Self { Self { - patch_size: DEFAULT_PATCH_SIZE, - merge_size: DEFAULT_MERGE_SIZE, - min_pixels: DEFAULT_MIN_PIXELS, - max_pixels: DEFAULT_MAX_PIXELS, - temporal_patch_size: DEFAULT_TEMPORAL_PATCH_SIZE, + inner: QwenVLProcessorBase::new(QwenVLConfig { + patch_size: DEFAULT_PATCH_SIZE, + merge_size: DEFAULT_MERGE_SIZE, + min_pixels: DEFAULT_MIN_PIXELS, + max_pixels: DEFAULT_MAX_PIXELS, + temporal_patch_size: DEFAULT_TEMPORAL_PATCH_SIZE, + mean: CLIP_MEAN, + std: CLIP_STD, + model_name: "qwen2-vl", + }), } } @@ -117,155 +102,94 @@ impl Qwen2VLProcessor { temporal_patch_size: usize, ) -> Self { Self { - patch_size, - merge_size, - min_pixels, - max_pixels, - temporal_patch_size, + inner: QwenVLProcessorBase::new(QwenVLConfig { + patch_size, + merge_size, + min_pixels, + max_pixels, + temporal_patch_size, + mean: CLIP_MEAN, + std: CLIP_STD, + model_name: "qwen2-vl", + }), } } /// Create a processor from preprocessor config. pub fn from_preprocessor_config(config: &PreProcessorConfig) -> Self { Self { - patch_size: config.patch_size.unwrap_or(DEFAULT_PATCH_SIZE), - merge_size: config.merge_size.unwrap_or(DEFAULT_MERGE_SIZE), - min_pixels: config.min_pixels.unwrap_or(DEFAULT_MIN_PIXELS), - max_pixels: config.max_pixels.unwrap_or(DEFAULT_MAX_PIXELS), - temporal_patch_size: config - .temporal_patch_size - .unwrap_or(DEFAULT_TEMPORAL_PATCH_SIZE), + inner: QwenVLProcessorBase::new(QwenVLConfig { + patch_size: config.patch_size.unwrap_or(DEFAULT_PATCH_SIZE), + merge_size: config.merge_size.unwrap_or(DEFAULT_MERGE_SIZE), + min_pixels: config.min_pixels.unwrap_or(DEFAULT_MIN_PIXELS), + max_pixels: config.max_pixels.unwrap_or(DEFAULT_MAX_PIXELS), + temporal_patch_size: config + .temporal_patch_size + .unwrap_or(DEFAULT_TEMPORAL_PATCH_SIZE), + mean: CLIP_MEAN, + std: CLIP_STD, + model_name: "qwen2-vl", + }), } } + /// Get the patch size. + pub fn patch_size(&self) -> usize { + self.inner.patch_size() + } + + /// Get the merge size. + pub fn merge_size(&self) -> usize { + self.inner.merge_size() + } + + /// Get the minimum pixels. + pub fn min_pixels(&self) -> usize { + self.inner.min_pixels() + } + + /// Get the maximum pixels. + pub fn max_pixels(&self) -> usize { + self.inner.max_pixels() + } + + /// Get the temporal patch size. + pub fn temporal_patch_size(&self) -> usize { + self.inner.temporal_patch_size() + } + /// Get the factor for dimension alignment. - /// - /// Dimensions must be divisible by (patch_size * merge_size). #[inline] pub fn get_factor(&self) -> usize { - self.patch_size * self.merge_size + self.inner.get_factor() } /// Smart resize algorithm for Qwen2-VL. - /// - /// Resizes image dimensions to fit within min/max pixel bounds while: - /// - Preserving aspect ratio - /// - Aligning to (patch_size * merge_size) boundaries - /// - /// # Arguments - /// * `height` - Original image height - /// * `width` - Original image width - /// - /// # Returns - /// (new_height, new_width) or error if aspect ratio is too extreme - /// - /// # Errors - /// - If height or width is smaller than the factor - /// - If aspect ratio exceeds 200:1 pub fn smart_resize( &self, height: usize, width: usize, ) -> Result<(usize, usize), TransformError> { - let factor = self.get_factor(); - - // Validate minimum dimensions - if height < factor || width < factor { - return Err(TransformError::InvalidShape { - expected: format!("dimensions >= {} (patch_size * merge_size)", factor), - actual: vec![height, width], - }); - } - - // Validate aspect ratio - let max_dim = height.max(width) as f64; - let min_dim = height.min(width) as f64; - let aspect_ratio = max_dim / min_dim; - if aspect_ratio > 200.0 { - return Err(TransformError::InvalidShape { - expected: "aspect ratio < 200:1".to_string(), - actual: vec![height, width], - }); - } - - // Round to nearest factor multiple - let mut h_bar = (height as f64 / factor as f64).round() as usize * factor; - let mut w_bar = (width as f64 / factor as f64).round() as usize * factor; - - // Ensure minimum size - h_bar = h_bar.max(factor); - w_bar = w_bar.max(factor); - - // Scale down if exceeding max_pixels - if h_bar * w_bar > self.max_pixels { - let beta = ((height * width) as f64 / self.max_pixels as f64).sqrt(); - h_bar = ((height as f64 / beta / factor as f64).floor() as usize) * factor; - w_bar = ((width as f64 / beta / factor as f64).floor() as usize) * factor; - // Ensure minimum size after scaling down - h_bar = h_bar.max(factor); - w_bar = w_bar.max(factor); - } - // Scale up if below min_pixels - else if h_bar * w_bar < self.min_pixels { - let beta = (self.min_pixels as f64 / (height * width) as f64).sqrt(); - h_bar = ((height as f64 * beta / factor as f64).ceil() as usize) * factor; - w_bar = ((width as f64 * beta / factor as f64).ceil() as usize) * factor; - } - - Ok((h_bar, w_bar)) + self.inner.smart_resize(height, width) } /// Calculate the grid dimensions (T, H, W) for an image. - /// - /// For single images, T=1. For video, T = num_frames / temporal_patch_size. - /// - /// # Arguments - /// * `height` - Resized image height - /// * `width` - Resized image width - /// * `num_frames` - Number of frames (1 for images) - /// - /// # Returns - /// (grid_t, grid_h, grid_w) pub fn calculate_grid_thw( &self, height: usize, width: usize, num_frames: usize, ) -> (usize, usize, usize) { - let grid_t = num_frames.max(self.temporal_patch_size) / self.temporal_patch_size; - let grid_h = height / self.patch_size; - let grid_w = width / self.patch_size; - (grid_t, grid_h, grid_w) + self.inner.calculate_grid_thw(height, width, num_frames) } /// Calculate the number of image tokens after merge. - /// - /// tokens = (grid_t * grid_h * grid_w) / merge_size² pub fn calculate_tokens_from_grid(&self, grid_t: usize, grid_h: usize, grid_w: usize) -> usize { - (grid_t * grid_h * grid_w) / (self.merge_size * self.merge_size) + self.inner + .calculate_tokens_from_grid(grid_t, grid_h, grid_w) } /// Reshape pixel values from [C, H, W] to flattened patches format. - /// - /// This matches the HuggingFace Qwen2VLImageProcessor output format: - /// `(num_patches, patch_features)` where: - /// - num_patches = grid_t * grid_h * grid_w - /// - patch_features = C * temporal_patch_size * patch_size * patch_size - /// - /// The transformation follows these steps (matching HuggingFace exactly): - /// 1. Start with [C, H, W] tensor, expand to [temporal, C, H, W] - /// 2. Reshape to [grid_t, temporal, C, grid_h/merge, merge, patch, grid_w/merge, merge, patch] - /// 3. Permute to [grid_t, grid_h/merge, grid_w/merge, merge, merge, C, temporal, patch, patch] - /// 4. Flatten to [num_patches, patch_features] - /// - /// # Arguments - /// * `tensor` - Input tensor of shape [C, H, W] - /// * `grid_t` - Temporal grid size (1 for images) - /// * `grid_h` - Height grid size (H / patch_size) - /// * `grid_w` - Width grid size (W / patch_size) - /// - /// # Returns - /// Flattened patches as Vec with shape semantics (num_patches, patch_features) pub fn reshape_to_patches( &self, tensor: &Array3, @@ -273,93 +197,26 @@ impl Qwen2VLProcessor { grid_h: usize, grid_w: usize, ) -> Vec { - use ndarray::IxDyn; + self.inner + .reshape_to_patches(tensor, grid_t, grid_h, grid_w) + } +} - let channel = tensor.shape()[0]; - let height = tensor.shape()[1]; - let width = tensor.shape()[2]; +impl Deref for Qwen2VLProcessor { + type Target = QwenVLProcessorBase; - let patch_size = self.patch_size; - let merge_size = self.merge_size; - let temporal_patch_size = self.temporal_patch_size; - - // Verify dimensions match expected grid - debug_assert_eq!( - height, - grid_h * patch_size, - "Height must match grid_h * patch_size" - ); - debug_assert_eq!( - width, - grid_w * patch_size, - "Width must match grid_w * patch_size" - ); - - // Step 1: Expand temporal dimension by replicating the frame - // [C, H, W] -> [temporal_patch_size, C, H, W] - let expanded = tensor - .view() - .insert_axis(ndarray::Axis(0)) - .broadcast((temporal_patch_size, channel, height, width)) - .expect("Broadcast failed") - .to_owned(); - - // Step 2: Reshape to split spatial dimensions into grid and patch components - // [temporal, C, H, W] -> [grid_t, temporal, C, grid_h/merge, merge, patch, grid_w/merge, merge, patch] - // - // Note: For images, grid_t=1 and we have temporal_patch_size frames (replicated) - // HF reshape: [grid_t, temporal, C, grid_h/merge, merge, patch, grid_w/merge, merge, patch] - let grid_h_merged = grid_h / merge_size; - let grid_w_merged = grid_w / merge_size; - - // Use IxDyn for 9-dimensional reshape (ndarray only supports up to Ix6 for fixed dims) - let shape_9d = IxDyn(&[ - grid_t, - temporal_patch_size, - channel, - grid_h_merged, - merge_size, - patch_size, - grid_w_merged, - merge_size, - patch_size, - ]); - - let reshaped = expanded - .into_shape_with_order(shape_9d) - .expect("Reshape failed"); - - // Step 3: Permute axes to match HuggingFace output order - // From: [grid_t, temporal, C, grid_h/merge, merge, patch, grid_w/merge, merge, patch] - // [ 0 , 1 , 2, 3 , 4 , 5 , 6 , 7 , 8 ] - // To: [grid_t, grid_h/merge, grid_w/merge, merge, merge, C, temporal, patch, patch] - // [ 0 , 3 , 6 , 4 , 7 , 2, 1 , 5 , 8 ] - let permuted = reshaped.permuted_axes(&[0, 3, 6, 4, 7, 2, 1, 5, 8][..]); - - // Step 4: Flatten to [num_patches, patch_features] - // num_patches = grid_t * grid_h * grid_w = grid_t * (grid_h/merge * merge) * (grid_w/merge * merge) - // patch_features = C * temporal * patch * patch - let num_patches = grid_t * grid_h * grid_w; - let patch_features = channel * temporal_patch_size * patch_size * patch_size; - - // Make contiguous and flatten - let contiguous = permuted.as_standard_layout().into_owned(); - let flat = contiguous - .into_shape_with_order(IxDyn(&[num_patches, patch_features])) - .expect("Final reshape failed"); - - let (vec, _offset) = flat.into_raw_vec_and_offset(); - vec + fn deref(&self) -> &Self::Target { + &self.inner } } impl ImagePreProcessor for Qwen2VLProcessor { fn default_mean(&self) -> [f64; 3] { - CLIP_MEAN + self.inner.default_mean() } fn default_std(&self) -> [f64; 3] { - CLIP_STD + self.inner.default_std() } fn preprocess( @@ -367,109 +224,19 @@ impl ImagePreProcessor for Qwen2VLProcessor { images: &[DynamicImage], config: &PreProcessorConfig, ) -> Result { - if images.is_empty() { - return Err(TransformError::EmptyBatch); - } - - // Store original sizes - let image_sizes: Vec<(u32, u32)> = images.iter().map(|img| img.dimensions()).collect(); - - // For Qwen2-VL, when batching multiple images, they may have different - // resized dimensions. We need to either: - // 1. Process each image separately and return individual tensors - // 2. Pad all images to the max dimensions in the batch - // - // Following mistral.rs approach: find max dimensions and resize all to that - - // First pass: calculate target dimensions for each image - let mut target_sizes = Vec::with_capacity(images.len()); - for image in images { - let (w, h) = image.dimensions(); - let (new_h, new_w) = self.smart_resize(h as usize, w as usize)?; - target_sizes.push((new_h, new_w)); - } - - // Find max height and width across all images - let max_height = target_sizes.iter().map(|(h, _)| *h).max().unwrap_or(0); - let max_width = target_sizes.iter().map(|(_, w)| *w).max().unwrap_or(0); - - // Process each image with uniform max dimensions - let mean = config.get_image_mean(); - let std = config.get_image_std(); - let filter = pil_to_filter(config.resampling); - - let mut tensors = Vec::with_capacity(images.len()); - let mut grid_thw_data = Vec::with_capacity(images.len() * 3); - let mut num_img_tokens = Vec::with_capacity(images.len()); - - for (i, image) in images.iter().enumerate() { - let (target_h, target_w) = target_sizes[i]; - - // Resize to the target size for this image - let resized = if config.do_resize.unwrap_or(true) { - // For batching: resize to max dimensions to enable stacking - // The actual grid dimensions are based on individual target sizes - resize(image, max_width as u32, max_height as u32, filter) - } else { - image.clone() - }; - - // Convert to tensor - let mut tensor = to_tensor(&resized); - - // Normalize - if config.do_normalize.unwrap_or(true) { - normalize(&mut tensor, &mean, &std); - } - - tensors.push(tensor); - - // Grid dimensions are based on the individual image's target size - let (grid_t, grid_h, grid_w) = self.calculate_grid_thw(target_h, target_w, 1); - grid_thw_data.push(grid_t as u32); - grid_thw_data.push(grid_h as u32); - grid_thw_data.push(grid_w as u32); - - // Token count is based on individual grid - let tokens = self.calculate_tokens_from_grid(grid_t, grid_h, grid_w); - num_img_tokens.push(tokens); - } - - // Stack tensors into batch (now all same size) - let pixel_values = stack_batch(&tensors)?; - - // Create result with model-specific image_grid_thw - let result = PreprocessedImages::new(pixel_values, num_img_tokens, image_sizes).with_extra( - "image_grid_thw", - ModelSpecificValue::uint_2d(grid_thw_data, images.len(), 3), - ); - - Ok(result) + self.inner.preprocess(images, config) } - fn calculate_num_tokens(&self, width: u32, height: u32, _config: &PreProcessorConfig) -> usize { - // Calculate resized dimensions - let (new_height, new_width) = match self.smart_resize(height as usize, width as usize) { - Ok((h, w)) => (h, w), - Err(_) => { - // Fallback: use minimum size - let factor = self.get_factor(); - (factor, factor) - } - }; - - // Calculate grid and tokens - let (grid_t, grid_h, grid_w) = self.calculate_grid_thw(new_height, new_width, 1); - self.calculate_tokens_from_grid(grid_t, grid_h, grid_w) + fn calculate_num_tokens(&self, width: u32, height: u32, config: &PreProcessorConfig) -> usize { + self.inner.calculate_num_tokens(width, height, config) } fn model_name(&self) -> &'static str { - "qwen2-vl" + self.inner.model_name() } - fn get_processed_size(&self, _config: &PreProcessorConfig) -> Option<(u32, u32)> { - // Qwen2-VL has dynamic sizing, no fixed output size - None + fn get_processed_size(&self, config: &PreProcessorConfig) -> Option<(u32, u32)> { + self.inner.get_processed_size(config) } } @@ -478,6 +245,7 @@ mod tests { use image::{Rgb, RgbImage}; use super::*; + use crate::multimodal::vision::image_processor::ModelSpecificValue; fn create_test_image(width: u32, height: u32, color: Rgb) -> DynamicImage { DynamicImage::from(RgbImage::from_pixel(width, height, color)) @@ -486,10 +254,10 @@ mod tests { #[test] fn test_qwen2_vl_processor_default() { let processor = Qwen2VLProcessor::new(); - assert_eq!(processor.patch_size, 14); - assert_eq!(processor.merge_size, 2); - assert_eq!(processor.min_pixels, DEFAULT_MIN_PIXELS); - assert_eq!(processor.max_pixels, DEFAULT_MAX_PIXELS); + assert_eq!(processor.patch_size(), 14); + assert_eq!(processor.merge_size(), 2); + assert_eq!(processor.min_pixels(), DEFAULT_MIN_PIXELS); + assert_eq!(processor.max_pixels(), DEFAULT_MAX_PIXELS); assert_eq!(processor.get_factor(), 28); // 14 * 2 } @@ -505,8 +273,8 @@ mod tests { assert_eq!(w % 28, 0); // Should be within bounds - assert!(h * w >= processor.min_pixels); - assert!(h * w <= processor.max_pixels); + assert!(h * w >= processor.min_pixels()); + assert!(h * w <= processor.max_pixels()); } #[test] @@ -517,7 +285,7 @@ mod tests { let (h, w) = processor.smart_resize(3000, 3000).unwrap(); // Should be scaled down - assert!(h * w <= processor.max_pixels); + assert!(h * w <= processor.max_pixels()); assert_eq!(h % 28, 0); assert_eq!(w % 28, 0); } @@ -530,7 +298,7 @@ mod tests { let (h, w) = processor.smart_resize(100, 100).unwrap(); // Should be scaled up to min_pixels - assert!(h * w >= processor.min_pixels); + assert!(h * w >= processor.min_pixels()); assert_eq!(h % 28, 0); assert_eq!(w % 28, 0); } @@ -660,11 +428,11 @@ mod tests { let processor = Qwen2VLProcessor::from_preprocessor_config(&config); - assert_eq!(processor.patch_size, 16); - assert_eq!(processor.merge_size, 4); - assert_eq!(processor.min_pixels, 100000); - assert_eq!(processor.max_pixels, 500000); - assert_eq!(processor.temporal_patch_size, 4); + assert_eq!(processor.patch_size(), 16); + assert_eq!(processor.merge_size(), 4); + assert_eq!(processor.min_pixels(), 100000); + assert_eq!(processor.max_pixels(), 500000); + assert_eq!(processor.temporal_patch_size(), 4); } #[test] diff --git a/sgl-router/src/multimodal/vision/processors/qwen3_vl.rs b/sgl-router/src/multimodal/vision/processors/qwen3_vl.rs new file mode 100644 index 000000000..9410ecc71 --- /dev/null +++ b/sgl-router/src/multimodal/vision/processors/qwen3_vl.rs @@ -0,0 +1,446 @@ +//! Qwen3-VL family image processors. +//! +//! This module provides the Qwen3-VL processor which wraps the shared +//! `QwenVLProcessorBase` with Qwen3-VL specific default parameters. +//! +//! # Key Differences from Qwen2-VL +//! +//! - **Patch Size**: 16 (vs 14 in Qwen2-VL) +//! - **Factor**: 32 (patch_size * merge_size) (vs 28 in Qwen2-VL) +//! - **Normalization**: [0.5, 0.5, 0.5] mean/std (vs CLIP in Qwen2-VL) +//! +//! # Qwen3-VL Parameters +//! +//! - patch_size: 16 +//! - merge_size: 2 +//! - factor: 32 (patch_size * merge_size) +//! - normalization: [0.5, 0.5, 0.5] mean/std + +use std::ops::Deref; + +use image::DynamicImage; +use ndarray::Array3; + +use super::qwen_vl_base::{QwenVLConfig, QwenVLProcessorBase}; +use crate::multimodal::vision::{ + image_processor::{ImagePreProcessor, PreprocessedImages}, + preprocessor_config::PreProcessorConfig, + transforms::TransformError, +}; + +/// Qwen3-VL normalization mean values (simple [0.5, 0.5, 0.5]). +pub const QWEN3_MEAN: [f64; 3] = [0.5, 0.5, 0.5]; + +/// Qwen3-VL normalization std values (simple [0.5, 0.5, 0.5]). +pub const QWEN3_STD: [f64; 3] = [0.5, 0.5, 0.5]; + +/// Default minimum pixels for Qwen3-VL +/// This corresponds to shortest_edge = 65536 from HF config +pub const DEFAULT_MIN_PIXELS: usize = 65536; + +/// Default maximum pixels for Qwen3-VL +/// This corresponds to longest_edge = 16777216 from HF config +pub const DEFAULT_MAX_PIXELS: usize = 16777216; + +/// Default patch size for Qwen3-VL (16, vs 14 in Qwen2-VL) +pub const DEFAULT_PATCH_SIZE: usize = 16; + +/// Default merge size for token reduction +pub const DEFAULT_MERGE_SIZE: usize = 2; + +/// Default temporal patch size (for video frames) +pub const DEFAULT_TEMPORAL_PATCH_SIZE: usize = 2; + +/// Qwen3-VL image processor. +/// +/// This is a thin wrapper around `QwenVLProcessorBase` with Qwen3-VL +/// specific default parameters: +/// - patch_size: 16 +/// - merge_size: 2 +/// - [0.5, 0.5, 0.5] normalization mean/std +#[derive(Debug, Clone)] +pub struct Qwen3VLProcessor { + inner: QwenVLProcessorBase, +} + +impl Default for Qwen3VLProcessor { + fn default() -> Self { + Self::new() + } +} + +impl Qwen3VLProcessor { + /// Create a new Qwen3-VL processor with default settings. + /// + /// Defaults: + /// - patch_size: 16 + /// - merge_size: 2 + /// - min_pixels: 65,536 + /// - max_pixels: 16,777,216 + /// - temporal_patch_size: 2 + /// - normalization: [0.5, 0.5, 0.5] mean/std + pub fn new() -> Self { + Self { + inner: QwenVLProcessorBase::new(QwenVLConfig { + patch_size: DEFAULT_PATCH_SIZE, + merge_size: DEFAULT_MERGE_SIZE, + min_pixels: DEFAULT_MIN_PIXELS, + max_pixels: DEFAULT_MAX_PIXELS, + temporal_patch_size: DEFAULT_TEMPORAL_PATCH_SIZE, + mean: QWEN3_MEAN, + std: QWEN3_STD, + model_name: "qwen3-vl", + }), + } + } + + /// Create a processor with custom settings. + pub fn with_config( + patch_size: usize, + merge_size: usize, + min_pixels: usize, + max_pixels: usize, + temporal_patch_size: usize, + ) -> Self { + Self { + inner: QwenVLProcessorBase::new(QwenVLConfig { + patch_size, + merge_size, + min_pixels, + max_pixels, + temporal_patch_size, + mean: QWEN3_MEAN, + std: QWEN3_STD, + model_name: "qwen3-vl", + }), + } + } + + /// Create a processor from preprocessor config. + pub fn from_preprocessor_config(config: &PreProcessorConfig) -> Self { + Self { + inner: QwenVLProcessorBase::new(QwenVLConfig { + patch_size: config.patch_size.unwrap_or(DEFAULT_PATCH_SIZE), + merge_size: config.merge_size.unwrap_or(DEFAULT_MERGE_SIZE), + min_pixels: config.min_pixels.unwrap_or(DEFAULT_MIN_PIXELS), + max_pixels: config.max_pixels.unwrap_or(DEFAULT_MAX_PIXELS), + temporal_patch_size: config + .temporal_patch_size + .unwrap_or(DEFAULT_TEMPORAL_PATCH_SIZE), + mean: QWEN3_MEAN, + std: QWEN3_STD, + model_name: "qwen3-vl", + }), + } + } + + /// Get the patch size. + pub fn patch_size(&self) -> usize { + self.inner.patch_size() + } + + /// Get the merge size. + pub fn merge_size(&self) -> usize { + self.inner.merge_size() + } + + /// Get the minimum pixels. + pub fn min_pixels(&self) -> usize { + self.inner.min_pixels() + } + + /// Get the maximum pixels. + pub fn max_pixels(&self) -> usize { + self.inner.max_pixels() + } + + /// Get the temporal patch size. + pub fn temporal_patch_size(&self) -> usize { + self.inner.temporal_patch_size() + } + + /// Get the factor for dimension alignment. + #[inline] + pub fn get_factor(&self) -> usize { + self.inner.get_factor() + } + + /// Smart resize algorithm for Qwen3-VL. + pub fn smart_resize( + &self, + height: usize, + width: usize, + ) -> Result<(usize, usize), TransformError> { + self.inner.smart_resize(height, width) + } + + /// Calculate the grid dimensions (T, H, W) for an image. + pub fn calculate_grid_thw( + &self, + height: usize, + width: usize, + num_frames: usize, + ) -> (usize, usize, usize) { + self.inner.calculate_grid_thw(height, width, num_frames) + } + + /// Calculate the number of image tokens after merge. + pub fn calculate_tokens_from_grid(&self, grid_t: usize, grid_h: usize, grid_w: usize) -> usize { + self.inner + .calculate_tokens_from_grid(grid_t, grid_h, grid_w) + } + + /// Reshape pixel values from [C, H, W] to flattened patches format. + pub fn reshape_to_patches( + &self, + tensor: &Array3, + grid_t: usize, + grid_h: usize, + grid_w: usize, + ) -> Vec { + self.inner + .reshape_to_patches(tensor, grid_t, grid_h, grid_w) + } +} + +impl Deref for Qwen3VLProcessor { + type Target = QwenVLProcessorBase; + + fn deref(&self) -> &Self::Target { + &self.inner + } +} + +impl ImagePreProcessor for Qwen3VLProcessor { + fn default_mean(&self) -> [f64; 3] { + self.inner.default_mean() + } + + fn default_std(&self) -> [f64; 3] { + self.inner.default_std() + } + + fn preprocess( + &self, + images: &[DynamicImage], + config: &PreProcessorConfig, + ) -> Result { + self.inner.preprocess(images, config) + } + + fn calculate_num_tokens(&self, width: u32, height: u32, config: &PreProcessorConfig) -> usize { + self.inner.calculate_num_tokens(width, height, config) + } + + fn model_name(&self) -> &'static str { + self.inner.model_name() + } + + fn get_processed_size(&self, config: &PreProcessorConfig) -> Option<(u32, u32)> { + self.inner.get_processed_size(config) + } +} + +#[cfg(test)] +mod tests { + use image::{Rgb, RgbImage}; + + use super::*; + use crate::multimodal::vision::image_processor::ModelSpecificValue; + + fn create_test_image(width: u32, height: u32, color: Rgb) -> DynamicImage { + DynamicImage::from(RgbImage::from_pixel(width, height, color)) + } + + #[test] + fn test_qwen3_vl_processor_default() { + let processor = Qwen3VLProcessor::new(); + assert_eq!(processor.patch_size(), 16); + assert_eq!(processor.merge_size(), 2); + assert_eq!(processor.min_pixels(), DEFAULT_MIN_PIXELS); + assert_eq!(processor.max_pixels(), DEFAULT_MAX_PIXELS); + assert_eq!(processor.get_factor(), 32); // 16 * 2 + } + + #[test] + fn test_smart_resize_within_bounds() { + let processor = Qwen3VLProcessor::new(); + + // Image that's within bounds + let (h, w) = processor.smart_resize(500, 500).unwrap(); + + // Should be aligned to factor (32) + assert_eq!(h % 32, 0); + assert_eq!(w % 32, 0); + + // Should be within bounds + assert!(h * w >= processor.min_pixels()); + assert!(h * w <= processor.max_pixels()); + } + + #[test] + fn test_smart_resize_aspect_ratio_preserved() { + let processor = Qwen3VLProcessor::new(); + + // 2:1 aspect ratio + let (h, w) = processor.smart_resize(400, 800).unwrap(); + + // Aspect ratio should be approximately preserved + let original_ratio = 800.0 / 400.0; + let new_ratio = w as f64 / h as f64; + assert!((new_ratio - original_ratio).abs() < 0.5); + } + + #[test] + fn test_smart_resize_extreme_aspect_ratio_error() { + let processor = Qwen3VLProcessor::new(); + + // 300:1 aspect ratio - should fail + let result = processor.smart_resize(100, 30000); + assert!(result.is_err()); + } + + #[test] + fn test_smart_resize_too_small_dimension_error() { + let processor = Qwen3VLProcessor::new(); + + // Dimension smaller than factor (32) + let result = processor.smart_resize(10, 100); + assert!(result.is_err()); + } + + #[test] + fn test_calculate_grid_thw_image() { + let processor = Qwen3VLProcessor::new(); + + // 480x640 image + let (t, h, w) = processor.calculate_grid_thw(480, 640, 1); + + assert_eq!(t, 1); // Single image + assert_eq!(h, 480 / 16); // 30 + assert_eq!(w, 640 / 16); // 40 + } + + #[test] + fn test_calculate_tokens() { + let processor = Qwen3VLProcessor::new(); + + // With merge_size=2, tokens = (t * h * w) / 4 + let tokens = processor.calculate_tokens_from_grid(1, 30, 40); + assert_eq!(tokens, (30 * 40) / 4); // 300 + } + + #[test] + fn test_qwen3_vl_preprocess() { + let processor = Qwen3VLProcessor::new(); + let config = PreProcessorConfig { + do_resize: Some(true), + do_normalize: Some(true), + image_mean: Some(QWEN3_MEAN.to_vec()), + image_std: Some(QWEN3_STD.to_vec()), + patch_size: Some(16), + merge_size: Some(2), + min_pixels: Some(DEFAULT_MIN_PIXELS), + max_pixels: Some(DEFAULT_MAX_PIXELS), + ..Default::default() + }; + + let image = create_test_image(640, 480, Rgb([128, 128, 128])); + let result = processor.preprocess(&[image], &config).unwrap(); + + assert_eq!(result.batch_size(), 1); + + // Check pixel values are normalized + let flat = result.pixel_values_flat(); + // After normalization with [0.5, 0.5, 0.5] mean/std: + // (0.5 - 0.5) / 0.5 = 0.0 for gray + // Values should be in [-1, 1] range + assert!(flat.iter().all(|&v| (-1.5..=1.5).contains(&v))); + + // Check image_grid_thw is present + assert!(result.model_specific.contains_key("image_grid_thw")); + + // Verify token count is reasonable + assert!(result.num_img_tokens[0] > 0); + } + + #[test] + fn test_qwen3_vl_preprocess_multiple() { + let processor = Qwen3VLProcessor::new(); + let config = PreProcessorConfig { + image_mean: Some(QWEN3_MEAN.to_vec()), + image_std: Some(QWEN3_STD.to_vec()), + ..Default::default() + }; + + let images = vec![ + create_test_image(640, 480, Rgb([100, 100, 100])), + create_test_image(480, 640, Rgb([150, 150, 150])), + ]; + + let result = processor.preprocess(&images, &config).unwrap(); + + // Both images processed + assert_eq!(result.image_sizes.len(), 2); + assert_eq!(result.num_img_tokens.len(), 2); + + // Check grid_thw shape + if let Some(ModelSpecificValue::UintTensor { data, shape }) = + result.model_specific.get("image_grid_thw") + { + assert_eq!(shape, &[2, 3]); // 2 images, 3 values (T, H, W) each + assert_eq!(data.len(), 6); + } else { + panic!("Expected image_grid_thw to be UintTensor"); + } + } + + #[test] + fn test_qwen3_vl_from_config() { + let config = PreProcessorConfig { + patch_size: Some(16), + merge_size: Some(4), + min_pixels: Some(100000), + max_pixels: Some(500000), + temporal_patch_size: Some(4), + ..Default::default() + }; + + let processor = Qwen3VLProcessor::from_preprocessor_config(&config); + + assert_eq!(processor.patch_size(), 16); + assert_eq!(processor.merge_size(), 4); + assert_eq!(processor.min_pixels(), 100000); + assert_eq!(processor.max_pixels(), 500000); + assert_eq!(processor.temporal_patch_size(), 4); + } + + #[test] + fn test_model_name() { + let processor = Qwen3VLProcessor::new(); + assert_eq!(processor.model_name(), "qwen3-vl"); + } + + #[test] + fn test_default_mean_std() { + let processor = Qwen3VLProcessor::new(); + assert_eq!(processor.default_mean(), QWEN3_MEAN); + assert_eq!(processor.default_std(), QWEN3_STD); + } + + #[test] + fn test_qwen3_vs_qwen2_differences() { + // Verify the key differences from Qwen2-VL + let processor = Qwen3VLProcessor::new(); + + // Qwen3-VL uses patch_size=16 (vs 14 in Qwen2) + assert_eq!(processor.patch_size(), 16); + + // Factor is 32 (vs 28 in Qwen2) + assert_eq!(processor.get_factor(), 32); + + // Mean/std are [0.5, 0.5, 0.5] (vs CLIP values in Qwen2) + assert_eq!(processor.default_mean(), [0.5, 0.5, 0.5]); + assert_eq!(processor.default_std(), [0.5, 0.5, 0.5]); + } +} diff --git a/sgl-router/src/multimodal/vision/processors/qwen_vl_base.rs b/sgl-router/src/multimodal/vision/processors/qwen_vl_base.rs new file mode 100644 index 000000000..fa2da2983 --- /dev/null +++ b/sgl-router/src/multimodal/vision/processors/qwen_vl_base.rs @@ -0,0 +1,475 @@ +//! Shared base implementation for Qwen VL family image processors. +//! +//! This module provides a generic processor that handles the common logic +//! for Qwen2-VL, Qwen2.5-VL, and Qwen3-VL models. The specific variants +//! differ only in their default parameters (patch_size, normalization values). +//! +//! # Processing Pipeline +//! +//! 1. Validate aspect ratio (must be < 200:1) +//! 2. Smart resize to fit within min/max pixel bounds +//! 3. Align dimensions to (patch_size * merge_size) boundary +//! 4. Convert to tensor and normalize +//! 5. Reshape into patches for the vision encoder +//! +//! # Token Calculation +//! +//! ```text +//! grid_t = 1 (for images, temporal dimension is 1) +//! grid_h = resized_height / patch_size +//! grid_w = resized_width / patch_size +//! num_tokens = (grid_t * grid_h * grid_w) / merge_size² +//! ``` + +use image::{DynamicImage, GenericImageView}; +use ndarray::Array3; + +use crate::multimodal::vision::{ + image_processor::{ImagePreProcessor, ModelSpecificValue, PreprocessedImages}, + preprocessor_config::PreProcessorConfig, + transforms::{normalize, pil_to_filter, resize, stack_batch, to_tensor, TransformError}, +}; + +/// Configuration for a Qwen VL processor variant. +#[derive(Debug, Clone)] +pub struct QwenVLConfig { + /// Vision encoder patch size + pub patch_size: usize, + /// Merge size for token reduction + pub merge_size: usize, + /// Minimum total pixels allowed + pub min_pixels: usize, + /// Maximum total pixels allowed + pub max_pixels: usize, + /// Temporal patch size for video + pub temporal_patch_size: usize, + /// Normalization mean values + pub mean: [f64; 3], + /// Normalization std values + pub std: [f64; 3], + /// Model name for identification + pub model_name: &'static str, +} + +/// Generic Qwen VL image processor. +/// +/// This struct implements the shared preprocessing logic for all Qwen VL +/// model variants. Each variant (Qwen2-VL, Qwen3-VL, etc.) uses this with +/// different configuration values. +#[derive(Debug, Clone)] +pub struct QwenVLProcessorBase { + config: QwenVLConfig, +} + +impl QwenVLProcessorBase { + /// Create a new processor with the given configuration. + pub fn new(config: QwenVLConfig) -> Self { + Self { config } + } + + /// Get the patch size. + pub fn patch_size(&self) -> usize { + self.config.patch_size + } + + /// Get the merge size. + pub fn merge_size(&self) -> usize { + self.config.merge_size + } + + /// Get the minimum pixels. + pub fn min_pixels(&self) -> usize { + self.config.min_pixels + } + + /// Get the maximum pixels. + pub fn max_pixels(&self) -> usize { + self.config.max_pixels + } + + /// Get the temporal patch size. + pub fn temporal_patch_size(&self) -> usize { + self.config.temporal_patch_size + } + + /// Get the factor for dimension alignment. + /// + /// Dimensions must be divisible by (patch_size * merge_size). + #[inline] + pub fn get_factor(&self) -> usize { + self.config.patch_size * self.config.merge_size + } + + /// Smart resize algorithm for Qwen VL models. + /// + /// Resizes image dimensions to fit within min/max pixel bounds while: + /// - Preserving aspect ratio + /// - Aligning to (patch_size * merge_size) boundaries + /// + /// # Arguments + /// * `height` - Original image height + /// * `width` - Original image width + /// + /// # Returns + /// (new_height, new_width) or error if aspect ratio is too extreme + /// + /// # Errors + /// - If height or width is smaller than the factor + /// - If aspect ratio exceeds 200:1 + pub fn smart_resize( + &self, + height: usize, + width: usize, + ) -> Result<(usize, usize), TransformError> { + let factor = self.get_factor(); + + // Validate minimum dimensions + if height < factor || width < factor { + return Err(TransformError::InvalidShape { + expected: format!("dimensions >= {} (patch_size * merge_size)", factor), + actual: vec![height, width], + }); + } + + // Validate aspect ratio + let max_dim = height.max(width) as f64; + let min_dim = height.min(width) as f64; + let aspect_ratio = max_dim / min_dim; + if aspect_ratio > 200.0 { + return Err(TransformError::InvalidShape { + expected: "aspect ratio < 200:1".to_string(), + actual: vec![height, width], + }); + } + + // Round to nearest factor multiple + let mut h_bar = (height as f64 / factor as f64).round() as usize * factor; + let mut w_bar = (width as f64 / factor as f64).round() as usize * factor; + + // Ensure minimum size + h_bar = h_bar.max(factor); + w_bar = w_bar.max(factor); + + // Scale down if exceeding max_pixels + if h_bar * w_bar > self.config.max_pixels { + let beta = ((height * width) as f64 / self.config.max_pixels as f64).sqrt(); + h_bar = ((height as f64 / beta / factor as f64).floor() as usize) * factor; + w_bar = ((width as f64 / beta / factor as f64).floor() as usize) * factor; + // Ensure minimum size after scaling down + h_bar = h_bar.max(factor); + w_bar = w_bar.max(factor); + } + // Scale up if below min_pixels + else if h_bar * w_bar < self.config.min_pixels { + let beta = (self.config.min_pixels as f64 / (height * width) as f64).sqrt(); + h_bar = ((height as f64 * beta / factor as f64).ceil() as usize) * factor; + w_bar = ((width as f64 * beta / factor as f64).ceil() as usize) * factor; + } + + Ok((h_bar, w_bar)) + } + + /// Calculate the grid dimensions (T, H, W) for an image. + /// + /// For single images, T=1. For video, T = num_frames / temporal_patch_size. + /// + /// # Arguments + /// * `height` - Resized image height + /// * `width` - Resized image width + /// * `num_frames` - Number of frames (1 for images) + /// + /// # Returns + /// (grid_t, grid_h, grid_w) + pub fn calculate_grid_thw( + &self, + height: usize, + width: usize, + num_frames: usize, + ) -> (usize, usize, usize) { + let grid_t = + num_frames.max(self.config.temporal_patch_size) / self.config.temporal_patch_size; + let grid_h = height / self.config.patch_size; + let grid_w = width / self.config.patch_size; + (grid_t, grid_h, grid_w) + } + + /// Calculate the number of image tokens after merge. + /// + /// tokens = (grid_t * grid_h * grid_w) / merge_size² + pub fn calculate_tokens_from_grid(&self, grid_t: usize, grid_h: usize, grid_w: usize) -> usize { + (grid_t * grid_h * grid_w) / (self.config.merge_size * self.config.merge_size) + } + + /// Reshape pixel values from [C, H, W] to flattened patches format. + /// + /// This matches the HuggingFace Qwen2VLImageProcessor output format: + /// `(num_patches, patch_features)` where: + /// - num_patches = grid_t * grid_h * grid_w + /// - patch_features = C * temporal_patch_size * patch_size * patch_size + /// + /// The transformation follows these steps (matching HuggingFace exactly): + /// 1. Start with [C, H, W] tensor, expand to [temporal, C, H, W] + /// 2. Reshape to [grid_t, temporal, C, grid_h/merge, merge, patch, grid_w/merge, merge, patch] + /// 3. Permute to [grid_t, grid_h/merge, grid_w/merge, merge, merge, C, temporal, patch, patch] + /// 4. Flatten to [num_patches, patch_features] + /// + /// # Arguments + /// * `tensor` - Input tensor of shape [C, H, W] + /// * `grid_t` - Temporal grid size (1 for images) + /// * `grid_h` - Height grid size (H / patch_size) + /// * `grid_w` - Width grid size (W / patch_size) + /// + /// # Returns + /// Flattened patches as Vec with shape semantics (num_patches, patch_features) + pub fn reshape_to_patches( + &self, + tensor: &Array3, + grid_t: usize, + grid_h: usize, + grid_w: usize, + ) -> Vec { + use ndarray::IxDyn; + + let channel = tensor.shape()[0]; + let height = tensor.shape()[1]; + let width = tensor.shape()[2]; + + let patch_size = self.config.patch_size; + let merge_size = self.config.merge_size; + let temporal_patch_size = self.config.temporal_patch_size; + + // Verify dimensions match expected grid + debug_assert_eq!( + height, + grid_h * patch_size, + "Height must match grid_h * patch_size" + ); + debug_assert_eq!( + width, + grid_w * patch_size, + "Width must match grid_w * patch_size" + ); + + // Step 1: Expand temporal dimension by replicating the frame + // [C, H, W] -> [temporal_patch_size, C, H, W] + let expanded = tensor + .view() + .insert_axis(ndarray::Axis(0)) + .broadcast((temporal_patch_size, channel, height, width)) + .expect("Broadcast failed") + .to_owned(); + + // Step 2: Reshape to split spatial dimensions into grid and patch components + // [temporal, C, H, W] -> [grid_t, temporal, C, grid_h/merge, merge, patch, grid_w/merge, merge, patch] + let grid_h_merged = grid_h / merge_size; + let grid_w_merged = grid_w / merge_size; + + // Use IxDyn for 9-dimensional reshape (ndarray only supports up to Ix6 for fixed dims) + let shape_9d = IxDyn(&[ + grid_t, + temporal_patch_size, + channel, + grid_h_merged, + merge_size, + patch_size, + grid_w_merged, + merge_size, + patch_size, + ]); + + let reshaped = expanded + .into_shape_with_order(shape_9d) + .expect("Reshape failed"); + + // Step 3: Permute axes to match HuggingFace output order + // From: [grid_t, temporal, C, grid_h/merge, merge, patch, grid_w/merge, merge, patch] + // [ 0 , 1 , 2, 3 , 4 , 5 , 6 , 7 , 8 ] + // To: [grid_t, grid_h/merge, grid_w/merge, merge, merge, C, temporal, patch, patch] + // [ 0 , 3 , 6 , 4 , 7 , 2, 1 , 5 , 8 ] + let permuted = reshaped.permuted_axes(&[0, 3, 6, 4, 7, 2, 1, 5, 8][..]); + + // Step 4: Flatten to [num_patches, patch_features] + let num_patches = grid_t * grid_h * grid_w; + let patch_features = channel * temporal_patch_size * patch_size * patch_size; + + // Make contiguous and flatten + let contiguous = permuted.as_standard_layout().into_owned(); + let flat = contiguous + .into_shape_with_order(IxDyn(&[num_patches, patch_features])) + .expect("Final reshape failed"); + + let (vec, _offset) = flat.into_raw_vec_and_offset(); + vec + } +} + +impl ImagePreProcessor for QwenVLProcessorBase { + fn default_mean(&self) -> [f64; 3] { + self.config.mean + } + + fn default_std(&self) -> [f64; 3] { + self.config.std + } + + fn preprocess( + &self, + images: &[DynamicImage], + config: &PreProcessorConfig, + ) -> Result { + if images.is_empty() { + return Err(TransformError::EmptyBatch); + } + + // Store original sizes + let image_sizes: Vec<(u32, u32)> = images.iter().map(|img| img.dimensions()).collect(); + + // First pass: calculate target dimensions for each image + let mut target_sizes = Vec::with_capacity(images.len()); + for image in images { + let (w, h) = image.dimensions(); + let (new_h, new_w) = self.smart_resize(h as usize, w as usize)?; + target_sizes.push((new_h, new_w)); + } + + // Find max height and width across all images + let max_height = target_sizes.iter().map(|(h, _)| *h).max().unwrap_or(0); + let max_width = target_sizes.iter().map(|(_, w)| *w).max().unwrap_or(0); + + // Process each image with uniform max dimensions + let mean = config.get_image_mean(); + let std = config.get_image_std(); + let filter = pil_to_filter(config.resampling); + + let mut tensors = Vec::with_capacity(images.len()); + let mut grid_thw_data = Vec::with_capacity(images.len() * 3); + let mut num_img_tokens = Vec::with_capacity(images.len()); + + for (i, image) in images.iter().enumerate() { + let (target_h, target_w) = target_sizes[i]; + + // Resize to the target size for this image + let resized = if config.do_resize.unwrap_or(true) { + // For batching: resize to max dimensions to enable stacking + resize(image, max_width as u32, max_height as u32, filter) + } else { + image.clone() + }; + + // Convert to tensor + let mut tensor = to_tensor(&resized); + + // Normalize + if config.do_normalize.unwrap_or(true) { + normalize(&mut tensor, &mean, &std); + } + + tensors.push(tensor); + + // Grid dimensions are based on the individual image's target size + let (grid_t, grid_h, grid_w) = self.calculate_grid_thw(target_h, target_w, 1); + grid_thw_data.push(grid_t as u32); + grid_thw_data.push(grid_h as u32); + grid_thw_data.push(grid_w as u32); + + // Token count is based on individual grid + let tokens = self.calculate_tokens_from_grid(grid_t, grid_h, grid_w); + num_img_tokens.push(tokens); + } + + // Stack tensors into batch (now all same size) + let pixel_values = stack_batch(&tensors)?; + + // Create result with model-specific image_grid_thw + let result = PreprocessedImages::new(pixel_values, num_img_tokens, image_sizes).with_extra( + "image_grid_thw", + ModelSpecificValue::uint_2d(grid_thw_data, images.len(), 3), + ); + + Ok(result) + } + + fn calculate_num_tokens(&self, width: u32, height: u32, _config: &PreProcessorConfig) -> usize { + // Calculate resized dimensions + let (new_height, new_width) = match self.smart_resize(height as usize, width as usize) { + Ok((h, w)) => (h, w), + Err(_) => { + // Fallback: use minimum size + let factor = self.get_factor(); + (factor, factor) + } + }; + + // Calculate grid and tokens + let (grid_t, grid_h, grid_w) = self.calculate_grid_thw(new_height, new_width, 1); + self.calculate_tokens_from_grid(grid_t, grid_h, grid_w) + } + + fn model_name(&self) -> &'static str { + self.config.model_name + } + + fn get_processed_size(&self, _config: &PreProcessorConfig) -> Option<(u32, u32)> { + // Qwen VL models have dynamic sizing, no fixed output size + None + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn create_test_config() -> QwenVLConfig { + QwenVLConfig { + patch_size: 14, + merge_size: 2, + min_pixels: 256 * 28 * 28, + max_pixels: 1280 * 28 * 28, + temporal_patch_size: 2, + mean: [0.5, 0.5, 0.5], + std: [0.5, 0.5, 0.5], + model_name: "test-qwen-vl", + } + } + + #[test] + fn test_qwen_vl_base_factor() { + let processor = QwenVLProcessorBase::new(create_test_config()); + assert_eq!(processor.get_factor(), 28); // 14 * 2 + } + + #[test] + fn test_smart_resize_within_bounds() { + let processor = QwenVLProcessorBase::new(create_test_config()); + let (h, w) = processor.smart_resize(500, 500).unwrap(); + + assert_eq!(h % 28, 0); + assert_eq!(w % 28, 0); + assert!(h * w >= processor.min_pixels()); + assert!(h * w <= processor.max_pixels()); + } + + #[test] + fn test_smart_resize_extreme_aspect_ratio_error() { + let processor = QwenVLProcessorBase::new(create_test_config()); + let result = processor.smart_resize(100, 30000); + assert!(result.is_err()); + } + + #[test] + fn test_calculate_grid_thw() { + let processor = QwenVLProcessorBase::new(create_test_config()); + let (t, h, w) = processor.calculate_grid_thw(448, 448, 1); + + assert_eq!(t, 1); + assert_eq!(h, 448 / 14); + assert_eq!(w, 448 / 14); + } + + #[test] + fn test_calculate_tokens() { + let processor = QwenVLProcessorBase::new(create_test_config()); + let tokens = processor.calculate_tokens_from_grid(1, 32, 32); + assert_eq!(tokens, (32 * 32) / 4); + } +} diff --git a/sgl-router/tests/vision_golden_tests.rs b/sgl-router/tests/vision_golden_tests.rs index cefc91a60..a9f81cbdd 100644 --- a/sgl-router/tests/vision_golden_tests.rs +++ b/sgl-router/tests/vision_golden_tests.rs @@ -7,6 +7,7 @@ //! - `llava/` - Standard CLIP processing (llava-hf/* models, no expand-to-square) //! - `llava_pad/` - Expand-to-square mode (liuhaotian/llava-* models, image_aspect_ratio=pad) //! - `qwen2_vl/` - Dynamic resolution with smart resize (Qwen/Qwen2-VL-* models) +//! - `qwen3_vl/` - Dynamic resolution with patch_size=16 and [0.5,0.5,0.5] norm (Qwen/Qwen3-VL-* models) //! //! To regenerate golden outputs: //! ```bash @@ -18,7 +19,7 @@ use std::{fs::File, io::Read, path::Path}; use ndarray::Array4; use sgl_model_gateway::multimodal::vision::{ image_processor::ModelSpecificValue, ImagePreProcessor, LlavaProcessor, PreProcessorConfig, - Qwen2VLProcessor, + Qwen2VLProcessor, Qwen3VLProcessor, }; /// Load a numpy .npz file and extract pixel_values @@ -384,3 +385,152 @@ fn test_qwen2_vl_golden_wide() { fn test_qwen2_vl_golden_small() { run_qwen2_vl_golden_test("small"); } + +// ============================================================================ +// Qwen3-VL tests +// ============================================================================ + +/// Run a Qwen3-VL golden test for a specific image. +/// +/// This test validates: +/// 1. image_grid_thw matches the HuggingFace output +/// 2. num_tokens calculation is correct +/// 3. Pixel values match after reshaping to patch format +/// +/// Key differences from Qwen2-VL: +/// - patch_size: 16 (vs 14) +/// - factor: 32 (vs 28) +/// - normalization: [0.5, 0.5, 0.5] (vs CLIP) +fn run_qwen3_vl_golden_test(image_name: &str) { + let golden_dir = Path::new("tests/fixtures/golden/qwen3_vl"); + let image_path = Path::new("tests/fixtures/images").join(format!("{}.jpg", image_name)); + + if !golden_dir.exists() || !image_path.exists() { + eprintln!( + "Golden test fixtures for qwen3_vl/{} not found, skipping test", + image_name + ); + eprintln!("Run: python scripts/generate_vision_golden.py --model qwen3_vl"); + return; + } + + let npz_path = golden_dir.join(format!("golden_{}.npz", image_name)); + let config = load_config(&golden_dir.join("preprocessor_config.json")); + + // Load golden values + let golden_grid_thw = load_golden_grid_thw(&npz_path); + let golden_num_tokens = load_golden_num_tokens(&npz_path); + let (golden_pixels, golden_shape) = load_golden_qwen2_vl_pixels(&npz_path); + + // Process image with our Rust processor + let image = image::open(&image_path).expect("Failed to open image"); + let processor = Qwen3VLProcessor::from_preprocessor_config(&config); + let result = processor + .preprocess(&[image], &config) + .expect("Processing failed"); + + // Extract image_grid_thw from result + let rust_grid_thw = match result.model_specific.get("image_grid_thw") { + Some(ModelSpecificValue::UintTensor { data, shape }) => { + assert_eq!(shape, &[1, 3], "Expected shape [1, 3] for single image"); + data.clone() + } + _ => panic!("Expected image_grid_thw in model_specific"), + }; + + // Compare grid dimensions + println!( + "qwen3_vl - {} image - Grid T H W: golden={:?}, rust={:?}", + image_name, golden_grid_thw, rust_grid_thw + ); + assert_eq!( + golden_grid_thw, rust_grid_thw, + "image_grid_thw mismatch for {}", + image_name + ); + + // Compare token counts + let rust_num_tokens = result.num_img_tokens[0]; + println!( + "qwen3_vl - {} image - Tokens: golden={}, rust={}", + image_name, golden_num_tokens, rust_num_tokens + ); + assert_eq!( + golden_num_tokens, rust_num_tokens, + "num_tokens mismatch for {}", + image_name + ); + + // Compare pixel values by reshaping our output to patch format + let grid_t = rust_grid_thw[0] as usize; + let grid_h = rust_grid_thw[1] as usize; + let grid_w = rust_grid_thw[2] as usize; + + // Get the tensor for the first image (batch index 0) + let pixel_values = &result.pixel_values; + let tensor_3d = pixel_values.index_axis(ndarray::Axis(0), 0).to_owned(); + + // Reshape to patches format + let rust_patches = processor.reshape_to_patches(&tensor_3d, grid_t, grid_h, grid_w); + + // Verify shapes match (Qwen3-VL has patch_size=16) + let expected_num_patches = grid_t * grid_h * grid_w; + let patch_size = config.patch_size.unwrap_or(16); + let temporal_patch_size = config.temporal_patch_size.unwrap_or(2); + let expected_patch_features = 3 * temporal_patch_size * patch_size * patch_size; + + println!( + "qwen3_vl - {} image - Patch shape: golden={:?}, rust=({}, {})", + image_name, golden_shape, expected_num_patches, expected_patch_features + ); + assert_eq!( + golden_shape, + (expected_num_patches, expected_patch_features), + "Patch shape mismatch" + ); + assert_eq!( + rust_patches.len(), + expected_num_patches * expected_patch_features, + "Rust patches size mismatch" + ); + + // Compare pixel values + let max_diff = rust_patches + .iter() + .zip(golden_pixels.iter()) + .map(|(r, g)| (r - g).abs()) + .fold(0.0f32, f32::max); + + println!( + "qwen3_vl - {} image - Max pixel diff: {:.6}", + image_name, max_diff + ); + + // Allow tolerance for floating point and interpolation differences + assert!( + max_diff < 0.02, + "Max pixel difference {} exceeds tolerance 0.02 for {}", + max_diff, + image_name + ); +} + +#[test] +fn test_qwen3_vl_golden_square() { + run_qwen3_vl_golden_test("square"); +} + +#[test] +fn test_qwen3_vl_golden_tall() { + run_qwen3_vl_golden_test("tall"); +} + +#[test] +fn test_qwen3_vl_golden_wide() { + run_qwen3_vl_golden_test("wide"); +} + +#[test] +fn test_qwen3_vl_golden_small() { + run_qwen3_vl_golden_test("small"); +}