chenchenghao/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[model-gateway] add qwen3_vl model image processor (#14377)

Browse Source
This commit is contained in:
Simo Lin
2025-12-03 17:16:37 -08:00
committed by GitHub
parent 388151053d
commit d42c167bfd
8 changed files with 1272 additions and 346 deletions
Download Patch File Download Diff File Expand all files Collapse all files

72
sgl-router/scripts/generate_vision_golden.py
View File

@@ -46,6 +46,11 @@ MODELS = {
"processor_class": "Qwen2VLImageProcessor",
"description": "Dynamic resolution with smart resize",
},
"qwen3_vl": {
"model_id": "Qwen/Qwen3-VL-8B-Instruct",
"processor_class": "Qwen2VLImageProcessorFast",
"description": "Dynamic resolution with patch_size=16 and [0.5,0.5,0.5] normalization",
},
}
# Default test images
@@ -283,6 +288,72 @@ def save_golden(model_key: str, image_name: str, data: dict, output_dir: str):
print(f" Saved: {config_path}")
def generate_golden_qwen3_vl(image_path: str, output_dir: str) -> dict:
"""Generate golden output for Qwen3-VL.
Qwen3-VL uses dynamic resolution with smart resize similar to Qwen2-VL
but with different parameters:
- patch_size: 16 (vs 14 in Qwen2-VL)
- factor: 32 (vs 28 in Qwen2-VL)
- normalization: [0.5, 0.5, 0.5] mean/std (vs CLIP values in Qwen2-VL)
Default parameters:
- patch_size: 16
- merge_size: 2
- temporal_patch_size: 2
"""
from transformers import AutoProcessor
processor = AutoProcessor.from_pretrained(
"Qwen/Qwen3-VL-8B-Instruct", trust_remote_code=True
)
image = Image.open(image_path).convert("RGB")
original_size = image.size
# Process image using the image processor directly
outputs = processor.image_processor(images=image, return_tensors="pt")
# Convert to numpy for saving
pixel_values = outputs["pixel_values"].numpy()
image_grid_thw = outputs.get("image_grid_thw")
if image_grid_thw is not None:
image_grid_thw = image_grid_thw.numpy()
# Get config values
img_processor = processor.image_processor
patch_size = getattr(img_processor, "patch_size", 16)
merge_size = getattr(img_processor, "merge_size", 2)
temporal_patch_size = getattr(img_processor, "temporal_patch_size", 2)
# Calculate number of tokens
if image_grid_thw is not None:
grid_thw = image_grid_thw[0]
num_tokens = int(np.prod(grid_thw) / (merge_size**2))
else:
num_tokens = None
result = {
"pixel_values": pixel_values,
"original_size": original_size,
"processor_config": img_processor.to_dict(),
}
if image_grid_thw is not None:
result["image_grid_thw"] = image_grid_thw
if num_tokens is not None:
result["num_tokens"] = num_tokens
# Add debug info
result["config_info"] = {
"patch_size": patch_size,
"merge_size": merge_size,
"temporal_patch_size": temporal_patch_size,
}
return result
def generate_for_model(model_key: str, image_paths: list, output_dir: str):
"""Generate golden outputs for a specific model."""
print(f"\nGenerating golden outputs for {model_key}...")
@@ -292,6 +363,7 @@ def generate_for_model(model_key: str, image_paths: list, output_dir: str):
"llava_pad": generate_golden_llava_pad,
"llava_next": generate_golden_llava_next,
"qwen2_vl": generate_golden_qwen2_vl,
"qwen3_vl": generate_golden_qwen3_vl,
}.get(model_key)
if generator_fn is None:

11
sgl-router/src/multimodal/vision/image_processor.rs
View File

@@ -266,6 +266,7 @@ impl ImageProcessorRegistry {
/// - `llava` -> LlavaProcessor (also matches llava-1.5, etc.)
/// - `qwen2-vl` -> Qwen2VLProcessor
/// - `qwen2.5-vl` -> Qwen2VLProcessor (same preprocessing as Qwen2-VL)
/// - `qwen3-vl` -> Qwen3VLProcessor (patch_size=16, [0.5,0.5,0.5] normalization)
pub fn with_defaults() -> Self {
let mut registry = Self::new();
@@ -282,6 +283,16 @@ impl ImageProcessorRegistry {
// Register standard LLaVA (matches llava-1.5, llava-v1.5, etc.)
registry.register("llava", Box::new(super::processors::LlavaProcessor::new()));
// Register Qwen3-VL first (more specific pattern - must match before qwen2)
registry.register(
"qwen3-vl",
Box::new(super::processors::Qwen3VLProcessor::new()),
);
registry.register(
"qwen3_vl",
Box::new(super::processors::Qwen3VLProcessor::new()),
);
// Register Qwen2-VL (matches Qwen/Qwen2-VL-*, etc.)
registry.register(
"qwen2-vl",

2
sgl-router/src/multimodal/vision/mod.rs
View File

@@ -39,5 +39,5 @@ pub use image_processor::{
ImagePreProcessor, ImageProcessorRegistry, ModelSpecificValue, PreprocessedImages,
};
pub use preprocessor_config::PreProcessorConfig;
pub use processors::{LlavaNextProcessor, LlavaProcessor, Qwen2VLProcessor};
pub use processors::{LlavaNextProcessor, LlavaProcessor, Qwen2VLProcessor, Qwen3VLProcessor};
pub use transforms::TransformError;

4
sgl-router/src/multimodal/vision/processors/mod.rs
View File

@@ -9,9 +9,13 @@
//! - **LLaVA-NeXT** (`llava`): Multi-crop anyres processing
//! - **Qwen2-VL** (`qwen2_vl`): Dynamic resolution with smart resizing
//! - **Qwen2.5-VL** (`qwen2_vl`): Same processor as Qwen2-VL (identical preprocessing)
//! - **Qwen3-VL** (`qwen3_vl`): Similar to Qwen2-VL but with patch_size=16 and [0.5,0.5,0.5] normalization
pub mod llava;
pub mod qwen2_vl;
pub mod qwen3_vl;
pub mod qwen_vl_base;
pub use llava::{ImageAspectRatio, LlavaNextProcessor, LlavaProcessor};
pub use qwen2_vl::Qwen2VLProcessor;
pub use qwen3_vl::Qwen3VLProcessor;

456
sgl-router/src/multimodal/vision/processors/qwen2_vl.rs
View File

@@ -1,7 +1,7 @@
//! Qwen2-VL family image processors.
//!
//! This module implements preprocessing for Qwen2-VL models, which use dynamic
//! resolution with smart resizing to maintain aspect ratio within pixel bounds.
//! This module provides the Qwen2-VL processor which wraps the shared
//! `QwenVLProcessorBase` with Qwen2-VL specific default parameters.
//!
//! # Key Features
//!
@@ -10,31 +10,23 @@
//! - **Dynamic Token Count**: Token count depends on actual image dimensions
//! - **image_grid_thw**: Returns (T, H, W) grid dimensions for position encoding
//!
//! # Processing Pipeline
//! # Qwen2-VL Parameters
//!
//! 1. Validate aspect ratio (must be < 200:1)
//! 2. Smart resize to fit within min/max pixel bounds
//! 3. Align dimensions to (patch_size * merge_size) boundary
//! 4. Convert to tensor and normalize with CLIP mean/std
//! 5. Reshape into patches for the vision encoder
//!
//! # Token Calculation
//!
//! For Qwen2-VL, the number of image tokens is:
//! ```text
//! grid_t = 1 (for images, temporal dimension is 1)
//! grid_h = resized_height / patch_size
//! grid_w = resized_width / patch_size
//! num_tokens = (grid_t * grid_h * grid_w) / merge_size²
//! ```
//! - patch_size: 14
//! - merge_size: 2
//! - factor: 28 (patch_size * merge_size)
//! - normalization: CLIP mean/std
use image::{DynamicImage, GenericImageView};
use std::ops::Deref;
use image::DynamicImage;
use ndarray::Array3;
use super::qwen_vl_base::{QwenVLConfig, QwenVLProcessorBase};
use crate::multimodal::vision::{
image_processor::{ImagePreProcessor, ModelSpecificValue, PreprocessedImages},
image_processor::{ImagePreProcessor, PreprocessedImages},
preprocessor_config::PreProcessorConfig,
transforms::{normalize, pil_to_filter, resize, stack_batch, to_tensor, TransformError},
transforms::TransformError,
};
/// CLIP normalization mean values used by Qwen2-VL models.
@@ -60,27 +52,14 @@ pub const DEFAULT_TEMPORAL_PATCH_SIZE: usize = 2;
/// Qwen2-VL image processor.
///
/// Implements dynamic resolution preprocessing with smart resizing that:
/// - Maintains aspect ratio
/// - Fits within configurable min/max pixel bounds
/// - Aligns to patch boundaries for efficient vision encoding
///
///
/// The processor returns `image_grid_thw` in the model-specific outputs,
/// which contains the (T, H, W) grid dimensions needed for rotary position
/// encoding in the Qwen2-VL model.
/// This is a thin wrapper around `QwenVLProcessorBase` with Qwen2-VL
/// specific default parameters:
/// - patch_size: 14
/// - merge_size: 2
/// - CLIP normalization mean/std
#[derive(Debug, Clone)]
pub struct Qwen2VLProcessor {
/// Vision encoder patch size (typically 14)
pub patch_size: usize,
/// Merge size for token reduction (typically 2)
pub merge_size: usize,
/// Minimum total pixels allowed
pub min_pixels: usize,
/// Maximum total pixels allowed
pub max_pixels: usize,
/// Temporal patch size for video (typically 2)
pub temporal_patch_size: usize,
inner: QwenVLProcessorBase,
}
impl Default for Qwen2VLProcessor {
@@ -98,13 +77,19 @@ impl Qwen2VLProcessor {
/// - min_pixels: 200,704 (256 * 28 * 28)
/// - max_pixels: 1,003,520 (1280 * 28 * 28)
/// - temporal_patch_size: 2
/// - normalization: CLIP mean/std
pub fn new() -> Self {
Self {
patch_size: DEFAULT_PATCH_SIZE,
merge_size: DEFAULT_MERGE_SIZE,
min_pixels: DEFAULT_MIN_PIXELS,
max_pixels: DEFAULT_MAX_PIXELS,
temporal_patch_size: DEFAULT_TEMPORAL_PATCH_SIZE,
inner: QwenVLProcessorBase::new(QwenVLConfig {
patch_size: DEFAULT_PATCH_SIZE,
merge_size: DEFAULT_MERGE_SIZE,
min_pixels: DEFAULT_MIN_PIXELS,
max_pixels: DEFAULT_MAX_PIXELS,
temporal_patch_size: DEFAULT_TEMPORAL_PATCH_SIZE,
mean: CLIP_MEAN,
std: CLIP_STD,
model_name: "qwen2-vl",
}),
}
}
@@ -117,155 +102,94 @@ impl Qwen2VLProcessor {
temporal_patch_size: usize,
) -> Self {
Self {
patch_size,
merge_size,
min_pixels,
max_pixels,
temporal_patch_size,
inner: QwenVLProcessorBase::new(QwenVLConfig {
patch_size,
merge_size,
min_pixels,
max_pixels,
temporal_patch_size,
mean: CLIP_MEAN,
std: CLIP_STD,
model_name: "qwen2-vl",
}),
}
}
/// Create a processor from preprocessor config.
pub fn from_preprocessor_config(config: &PreProcessorConfig) -> Self {
Self {
patch_size: config.patch_size.unwrap_or(DEFAULT_PATCH_SIZE),
merge_size: config.merge_size.unwrap_or(DEFAULT_MERGE_SIZE),
min_pixels: config.min_pixels.unwrap_or(DEFAULT_MIN_PIXELS),
max_pixels: config.max_pixels.unwrap_or(DEFAULT_MAX_PIXELS),
temporal_patch_size: config
.temporal_patch_size
.unwrap_or(DEFAULT_TEMPORAL_PATCH_SIZE),
inner: QwenVLProcessorBase::new(QwenVLConfig {
patch_size: config.patch_size.unwrap_or(DEFAULT_PATCH_SIZE),
merge_size: config.merge_size.unwrap_or(DEFAULT_MERGE_SIZE),
min_pixels: config.min_pixels.unwrap_or(DEFAULT_MIN_PIXELS),
max_pixels: config.max_pixels.unwrap_or(DEFAULT_MAX_PIXELS),
temporal_patch_size: config
.temporal_patch_size
.unwrap_or(DEFAULT_TEMPORAL_PATCH_SIZE),
mean: CLIP_MEAN,
std: CLIP_STD,
model_name: "qwen2-vl",
}),
}
}
/// Get the patch size.
pub fn patch_size(&self) -> usize {
self.inner.patch_size()
}
/// Get the merge size.
pub fn merge_size(&self) -> usize {
self.inner.merge_size()
}
/// Get the minimum pixels.
pub fn min_pixels(&self) -> usize {
self.inner.min_pixels()
}
/// Get the maximum pixels.
pub fn max_pixels(&self) -> usize {
self.inner.max_pixels()
}
/// Get the temporal patch size.
pub fn temporal_patch_size(&self) -> usize {
self.inner.temporal_patch_size()
}
/// Get the factor for dimension alignment.
///
/// Dimensions must be divisible by (patch_size * merge_size).
#[inline]
pub fn get_factor(&self) -> usize {
self.patch_size * self.merge_size
self.inner.get_factor()
}
/// Smart resize algorithm for Qwen2-VL.
///
/// Resizes image dimensions to fit within min/max pixel bounds while:
/// - Preserving aspect ratio
/// - Aligning to (patch_size * merge_size) boundaries
///
/// # Arguments
/// * `height` - Original image height
/// * `width` - Original image width
///
/// # Returns
/// (new_height, new_width) or error if aspect ratio is too extreme
///
/// # Errors
/// - If height or width is smaller than the factor
/// - If aspect ratio exceeds 200:1
pub fn smart_resize(
&self,
height: usize,
width: usize,
) -> Result<(usize, usize), TransformError> {
let factor = self.get_factor();
// Validate minimum dimensions
if height < factor || width < factor {
return Err(TransformError::InvalidShape {
expected: format!("dimensions >= {} (patch_size * merge_size)", factor),
actual: vec![height, width],
});
}
// Validate aspect ratio
let max_dim = height.max(width) as f64;
let min_dim = height.min(width) as f64;
let aspect_ratio = max_dim / min_dim;
if aspect_ratio > 200.0 {
return Err(TransformError::InvalidShape {
expected: "aspect ratio < 200:1".to_string(),
actual: vec![height, width],
});
}
// Round to nearest factor multiple
let mut h_bar = (height as f64 / factor as f64).round() as usize * factor;
let mut w_bar = (width as f64 / factor as f64).round() as usize * factor;
// Ensure minimum size
h_bar = h_bar.max(factor);
w_bar = w_bar.max(factor);
// Scale down if exceeding max_pixels
if h_bar * w_bar > self.max_pixels {
let beta = ((height * width) as f64 / self.max_pixels as f64).sqrt();
h_bar = ((height as f64 / beta / factor as f64).floor() as usize) * factor;
w_bar = ((width as f64 / beta / factor as f64).floor() as usize) * factor;
// Ensure minimum size after scaling down
h_bar = h_bar.max(factor);
w_bar = w_bar.max(factor);
}
// Scale up if below min_pixels
else if h_bar * w_bar < self.min_pixels {
let beta = (self.min_pixels as f64 / (height * width) as f64).sqrt();
h_bar = ((height as f64 * beta / factor as f64).ceil() as usize) * factor;
w_bar = ((width as f64 * beta / factor as f64).ceil() as usize) * factor;
}
Ok((h_bar, w_bar))
self.inner.smart_resize(height, width)
}
/// Calculate the grid dimensions (T, H, W) for an image.
///
/// For single images, T=1. For video, T = num_frames / temporal_patch_size.
///
/// # Arguments
/// * `height` - Resized image height
/// * `width` - Resized image width
/// * `num_frames` - Number of frames (1 for images)
///
/// # Returns
/// (grid_t, grid_h, grid_w)
pub fn calculate_grid_thw(
&self,
height: usize,
width: usize,
num_frames: usize,
) -> (usize, usize, usize) {
let grid_t = num_frames.max(self.temporal_patch_size) / self.temporal_patch_size;
let grid_h = height / self.patch_size;
let grid_w = width / self.patch_size;
(grid_t, grid_h, grid_w)
self.inner.calculate_grid_thw(height, width, num_frames)
}
/// Calculate the number of image tokens after merge.
///
/// tokens = (grid_t * grid_h * grid_w) / merge_size²
pub fn calculate_tokens_from_grid(&self, grid_t: usize, grid_h: usize, grid_w: usize) -> usize {
(grid_t * grid_h * grid_w) / (self.merge_size * self.merge_size)
self.inner
.calculate_tokens_from_grid(grid_t, grid_h, grid_w)
}
/// Reshape pixel values from [C, H, W] to flattened patches format.
///
/// This matches the HuggingFace Qwen2VLImageProcessor output format:
/// `(num_patches, patch_features)` where:
/// - num_patches = grid_t * grid_h * grid_w
/// - patch_features = C * temporal_patch_size * patch_size * patch_size
///
/// The transformation follows these steps (matching HuggingFace exactly):
/// 1. Start with [C, H, W] tensor, expand to [temporal, C, H, W]
/// 2. Reshape to [grid_t, temporal, C, grid_h/merge, merge, patch, grid_w/merge, merge, patch]
/// 3. Permute to [grid_t, grid_h/merge, grid_w/merge, merge, merge, C, temporal, patch, patch]
/// 4. Flatten to [num_patches, patch_features]
///
/// # Arguments
/// * `tensor` - Input tensor of shape [C, H, W]
/// * `grid_t` - Temporal grid size (1 for images)
/// * `grid_h` - Height grid size (H / patch_size)
/// * `grid_w` - Width grid size (W / patch_size)
///
/// # Returns
/// Flattened patches as Vec<f32> with shape semantics (num_patches, patch_features)
pub fn reshape_to_patches(
&self,
tensor: &Array3<f32>,
@@ -273,93 +197,26 @@ impl Qwen2VLProcessor {
grid_h: usize,
grid_w: usize,
) -> Vec<f32> {
use ndarray::IxDyn;
self.inner
.reshape_to_patches(tensor, grid_t, grid_h, grid_w)
}
}
let channel = tensor.shape()[0];
let height = tensor.shape()[1];
let width = tensor.shape()[2];
impl Deref for Qwen2VLProcessor {
type Target = QwenVLProcessorBase;
let patch_size = self.patch_size;
let merge_size = self.merge_size;
let temporal_patch_size = self.temporal_patch_size;
// Verify dimensions match expected grid
debug_assert_eq!(
height,
grid_h * patch_size,
"Height must match grid_h * patch_size"
);
debug_assert_eq!(
width,
grid_w * patch_size,
"Width must match grid_w * patch_size"
);
// Step 1: Expand temporal dimension by replicating the frame
// [C, H, W] -> [temporal_patch_size, C, H, W]
let expanded = tensor
.view()
.insert_axis(ndarray::Axis(0))
.broadcast((temporal_patch_size, channel, height, width))
.expect("Broadcast failed")
.to_owned();
// Step 2: Reshape to split spatial dimensions into grid and patch components
// [temporal, C, H, W] -> [grid_t, temporal, C, grid_h/merge, merge, patch, grid_w/merge, merge, patch]
//
// Note: For images, grid_t=1 and we have temporal_patch_size frames (replicated)
// HF reshape: [grid_t, temporal, C, grid_h/merge, merge, patch, grid_w/merge, merge, patch]
let grid_h_merged = grid_h / merge_size;
let grid_w_merged = grid_w / merge_size;
// Use IxDyn for 9-dimensional reshape (ndarray only supports up to Ix6 for fixed dims)
let shape_9d = IxDyn(&[
grid_t,
temporal_patch_size,
channel,
grid_h_merged,
merge_size,
patch_size,
grid_w_merged,
merge_size,
patch_size,
]);
let reshaped = expanded
.into_shape_with_order(shape_9d)
.expect("Reshape failed");
// Step 3: Permute axes to match HuggingFace output order
// From: [grid_t, temporal, C, grid_h/merge, merge, patch, grid_w/merge, merge, patch]
// [ 0 , 1 , 2, 3 , 4 , 5 , 6 , 7 , 8 ]
// To: [grid_t, grid_h/merge, grid_w/merge, merge, merge, C, temporal, patch, patch]
// [ 0 , 3 , 6 , 4 , 7 , 2, 1 , 5 , 8 ]
let permuted = reshaped.permuted_axes(&[0, 3, 6, 4, 7, 2, 1, 5, 8][..]);
// Step 4: Flatten to [num_patches, patch_features]
// num_patches = grid_t * grid_h * grid_w = grid_t * (grid_h/merge * merge) * (grid_w/merge * merge)
// patch_features = C * temporal * patch * patch
let num_patches = grid_t * grid_h * grid_w;
let patch_features = channel * temporal_patch_size * patch_size * patch_size;
// Make contiguous and flatten
let contiguous = permuted.as_standard_layout().into_owned();
let flat = contiguous
.into_shape_with_order(IxDyn(&[num_patches, patch_features]))
.expect("Final reshape failed");
let (vec, _offset) = flat.into_raw_vec_and_offset();
vec
fn deref(&self) -> &Self::Target {
&self.inner
}
}
impl ImagePreProcessor for Qwen2VLProcessor {
fn default_mean(&self) -> [f64; 3] {
CLIP_MEAN
self.inner.default_mean()
}
fn default_std(&self) -> [f64; 3] {
CLIP_STD
self.inner.default_std()
}
fn preprocess(
@@ -367,109 +224,19 @@ impl ImagePreProcessor for Qwen2VLProcessor {
images: &[DynamicImage],
config: &PreProcessorConfig,
) -> Result<PreprocessedImages, TransformError> {
if images.is_empty() {
return Err(TransformError::EmptyBatch);
}
// Store original sizes
let image_sizes: Vec<(u32, u32)> = images.iter().map(|img| img.dimensions()).collect();
// For Qwen2-VL, when batching multiple images, they may have different
// resized dimensions. We need to either:
// 1. Process each image separately and return individual tensors
// 2. Pad all images to the max dimensions in the batch
//
// Following mistral.rs approach: find max dimensions and resize all to that
// First pass: calculate target dimensions for each image
let mut target_sizes = Vec::with_capacity(images.len());
for image in images {
let (w, h) = image.dimensions();
let (new_h, new_w) = self.smart_resize(h as usize, w as usize)?;
target_sizes.push((new_h, new_w));
}
// Find max height and width across all images
let max_height = target_sizes.iter().map(|(h, _)| *h).max().unwrap_or(0);
let max_width = target_sizes.iter().map(|(_, w)| *w).max().unwrap_or(0);
// Process each image with uniform max dimensions
let mean = config.get_image_mean();
let std = config.get_image_std();
let filter = pil_to_filter(config.resampling);
let mut tensors = Vec::with_capacity(images.len());
let mut grid_thw_data = Vec::with_capacity(images.len() * 3);
let mut num_img_tokens = Vec::with_capacity(images.len());
for (i, image) in images.iter().enumerate() {
let (target_h, target_w) = target_sizes[i];
// Resize to the target size for this image
let resized = if config.do_resize.unwrap_or(true) {
// For batching: resize to max dimensions to enable stacking
// The actual grid dimensions are based on individual target sizes
resize(image, max_width as u32, max_height as u32, filter)
} else {
image.clone()
};
// Convert to tensor
let mut tensor = to_tensor(&resized);
// Normalize
if config.do_normalize.unwrap_or(true) {
normalize(&mut tensor, &mean, &std);
}
tensors.push(tensor);
// Grid dimensions are based on the individual image's target size
let (grid_t, grid_h, grid_w) = self.calculate_grid_thw(target_h, target_w, 1);
grid_thw_data.push(grid_t as u32);
grid_thw_data.push(grid_h as u32);
grid_thw_data.push(grid_w as u32);
// Token count is based on individual grid
let tokens = self.calculate_tokens_from_grid(grid_t, grid_h, grid_w);
num_img_tokens.push(tokens);
}
// Stack tensors into batch (now all same size)
let pixel_values = stack_batch(&tensors)?;
// Create result with model-specific image_grid_thw
let result = PreprocessedImages::new(pixel_values, num_img_tokens, image_sizes).with_extra(
"image_grid_thw",
ModelSpecificValue::uint_2d(grid_thw_data, images.len(), 3),
);
Ok(result)
self.inner.preprocess(images, config)
}
fn calculate_num_tokens(&self, width: u32, height: u32, _config: &PreProcessorConfig) -> usize {
// Calculate resized dimensions
let (new_height, new_width) = match self.smart_resize(height as usize, width as usize) {
Ok((h, w)) => (h, w),
Err(_) => {
// Fallback: use minimum size
let factor = self.get_factor();
(factor, factor)
}
};
// Calculate grid and tokens
let (grid_t, grid_h, grid_w) = self.calculate_grid_thw(new_height, new_width, 1);
self.calculate_tokens_from_grid(grid_t, grid_h, grid_w)
fn calculate_num_tokens(&self, width: u32, height: u32, config: &PreProcessorConfig) -> usize {
self.inner.calculate_num_tokens(width, height, config)
}
fn model_name(&self) -> &'static str {
"qwen2-vl"
self.inner.model_name()
}
fn get_processed_size(&self, _config: &PreProcessorConfig) -> Option<(u32, u32)> {
// Qwen2-VL has dynamic sizing, no fixed output size
None
fn get_processed_size(&self, config: &PreProcessorConfig) -> Option<(u32, u32)> {
self.inner.get_processed_size(config)
}
}
@@ -478,6 +245,7 @@ mod tests {
use image::{Rgb, RgbImage};
use super::*;
use crate::multimodal::vision::image_processor::ModelSpecificValue;
fn create_test_image(width: u32, height: u32, color: Rgb<u8>) -> DynamicImage {
DynamicImage::from(RgbImage::from_pixel(width, height, color))
@@ -486,10 +254,10 @@ mod tests {
#[test]
fn test_qwen2_vl_processor_default() {
let processor = Qwen2VLProcessor::new();
assert_eq!(processor.patch_size, 14);
assert_eq!(processor.merge_size, 2);
assert_eq!(processor.min_pixels, DEFAULT_MIN_PIXELS);
assert_eq!(processor.max_pixels, DEFAULT_MAX_PIXELS);
assert_eq!(processor.patch_size(), 14);
assert_eq!(processor.merge_size(), 2);
assert_eq!(processor.min_pixels(), DEFAULT_MIN_PIXELS);
assert_eq!(processor.max_pixels(), DEFAULT_MAX_PIXELS);
assert_eq!(processor.get_factor(), 28); // 14 * 2
}
@@ -505,8 +273,8 @@ mod tests {
assert_eq!(w % 28, 0);
// Should be within bounds
assert!(h * w >= processor.min_pixels);
assert!(h * w <= processor.max_pixels);
assert!(h * w >= processor.min_pixels());
assert!(h * w <= processor.max_pixels());
}
#[test]
@@ -517,7 +285,7 @@ mod tests {
let (h, w) = processor.smart_resize(3000, 3000).unwrap();
// Should be scaled down
assert!(h * w <= processor.max_pixels);
assert!(h * w <= processor.max_pixels());
assert_eq!(h % 28, 0);
assert_eq!(w % 28, 0);
}
@@ -530,7 +298,7 @@ mod tests {
let (h, w) = processor.smart_resize(100, 100).unwrap();
// Should be scaled up to min_pixels
assert!(h * w >= processor.min_pixels);
assert!(h * w >= processor.min_pixels());
assert_eq!(h % 28, 0);
assert_eq!(w % 28, 0);
}
@@ -660,11 +428,11 @@ mod tests {
let processor = Qwen2VLProcessor::from_preprocessor_config(&config);
assert_eq!(processor.patch_size, 16);
assert_eq!(processor.merge_size, 4);
assert_eq!(processor.min_pixels, 100000);
assert_eq!(processor.max_pixels, 500000);
assert_eq!(processor.temporal_patch_size, 4);
assert_eq!(processor.patch_size(), 16);
assert_eq!(processor.merge_size(), 4);
assert_eq!(processor.min_pixels(), 100000);
assert_eq!(processor.max_pixels(), 500000);
assert_eq!(processor.temporal_patch_size(), 4);
}
#[test]

446
sgl-router/src/multimodal/vision/processors/qwen3_vl.rs Normal file
View File

@@ -0,0 +1,446 @@
//! Qwen3-VL family image processors.
//!
//! This module provides the Qwen3-VL processor which wraps the shared
//! `QwenVLProcessorBase` with Qwen3-VL specific default parameters.
//!
//! # Key Differences from Qwen2-VL
//!
//! - **Patch Size**: 16 (vs 14 in Qwen2-VL)
//! - **Factor**: 32 (patch_size * merge_size) (vs 28 in Qwen2-VL)
//! - **Normalization**: [0.5, 0.5, 0.5] mean/std (vs CLIP in Qwen2-VL)
//!
//! # Qwen3-VL Parameters
//!
//! - patch_size: 16
//! - merge_size: 2
//! - factor: 32 (patch_size * merge_size)
//! - normalization: [0.5, 0.5, 0.5] mean/std
use std::ops::Deref;
use image::DynamicImage;
use ndarray::Array3;
use super::qwen_vl_base::{QwenVLConfig, QwenVLProcessorBase};
use crate::multimodal::vision::{
image_processor::{ImagePreProcessor, PreprocessedImages},
preprocessor_config::PreProcessorConfig,
transforms::TransformError,
};
/// Qwen3-VL normalization mean values (simple [0.5, 0.5, 0.5]).
pub const QWEN3_MEAN: [f64; 3] = [0.5, 0.5, 0.5];
/// Qwen3-VL normalization std values (simple [0.5, 0.5, 0.5]).
pub const QWEN3_STD: [f64; 3] = [0.5, 0.5, 0.5];
/// Default minimum pixels for Qwen3-VL
/// This corresponds to shortest_edge = 65536 from HF config
pub const DEFAULT_MIN_PIXELS: usize = 65536;
/// Default maximum pixels for Qwen3-VL
/// This corresponds to longest_edge = 16777216 from HF config
pub const DEFAULT_MAX_PIXELS: usize = 16777216;
/// Default patch size for Qwen3-VL (16, vs 14 in Qwen2-VL)
pub const DEFAULT_PATCH_SIZE: usize = 16;
/// Default merge size for token reduction
pub const DEFAULT_MERGE_SIZE: usize = 2;
/// Default temporal patch size (for video frames)
pub const DEFAULT_TEMPORAL_PATCH_SIZE: usize = 2;
/// Qwen3-VL image processor.
///
/// This is a thin wrapper around `QwenVLProcessorBase` with Qwen3-VL
/// specific default parameters:
/// - patch_size: 16
/// - merge_size: 2
/// - [0.5, 0.5, 0.5] normalization mean/std
#[derive(Debug, Clone)]
pub struct Qwen3VLProcessor {
inner: QwenVLProcessorBase,
}
impl Default for Qwen3VLProcessor {
fn default() -> Self {
Self::new()
}
}
impl Qwen3VLProcessor {
/// Create a new Qwen3-VL processor with default settings.
///
/// Defaults:
/// - patch_size: 16
/// - merge_size: 2
/// - min_pixels: 65,536
/// - max_pixels: 16,777,216
/// - temporal_patch_size: 2
/// - normalization: [0.5, 0.5, 0.5] mean/std
pub fn new() -> Self {
Self {
inner: QwenVLProcessorBase::new(QwenVLConfig {
patch_size: DEFAULT_PATCH_SIZE,
merge_size: DEFAULT_MERGE_SIZE,
min_pixels: DEFAULT_MIN_PIXELS,
max_pixels: DEFAULT_MAX_PIXELS,
temporal_patch_size: DEFAULT_TEMPORAL_PATCH_SIZE,
mean: QWEN3_MEAN,
std: QWEN3_STD,
model_name: "qwen3-vl",
}),
}
}
/// Create a processor with custom settings.
pub fn with_config(
patch_size: usize,
merge_size: usize,
min_pixels: usize,
max_pixels: usize,
temporal_patch_size: usize,
) -> Self {
Self {
inner: QwenVLProcessorBase::new(QwenVLConfig {
patch_size,
merge_size,
min_pixels,
max_pixels,
temporal_patch_size,
mean: QWEN3_MEAN,
std: QWEN3_STD,
model_name: "qwen3-vl",
}),
}
}
/// Create a processor from preprocessor config.
pub fn from_preprocessor_config(config: &PreProcessorConfig) -> Self {
Self {
inner: QwenVLProcessorBase::new(QwenVLConfig {
patch_size: config.patch_size.unwrap_or(DEFAULT_PATCH_SIZE),
merge_size: config.merge_size.unwrap_or(DEFAULT_MERGE_SIZE),
min_pixels: config.min_pixels.unwrap_or(DEFAULT_MIN_PIXELS),
max_pixels: config.max_pixels.unwrap_or(DEFAULT_MAX_PIXELS),
temporal_patch_size: config
.temporal_patch_size
.unwrap_or(DEFAULT_TEMPORAL_PATCH_SIZE),
mean: QWEN3_MEAN,
std: QWEN3_STD,
model_name: "qwen3-vl",
}),
}
}
/// Get the patch size.
pub fn patch_size(&self) -> usize {
self.inner.patch_size()
}
/// Get the merge size.
pub fn merge_size(&self) -> usize {
self.inner.merge_size()
}
/// Get the minimum pixels.
pub fn min_pixels(&self) -> usize {
self.inner.min_pixels()
}
/// Get the maximum pixels.
pub fn max_pixels(&self) -> usize {
self.inner.max_pixels()
}
/// Get the temporal patch size.
pub fn temporal_patch_size(&self) -> usize {
self.inner.temporal_patch_size()
}
/// Get the factor for dimension alignment.
#[inline]
pub fn get_factor(&self) -> usize {
self.inner.get_factor()
}
/// Smart resize algorithm for Qwen3-VL.
pub fn smart_resize(
&self,
height: usize,
width: usize,
) -> Result<(usize, usize), TransformError> {
self.inner.smart_resize(height, width)
}
/// Calculate the grid dimensions (T, H, W) for an image.
pub fn calculate_grid_thw(
&self,
height: usize,
width: usize,
num_frames: usize,
) -> (usize, usize, usize) {
self.inner.calculate_grid_thw(height, width, num_frames)
}
/// Calculate the number of image tokens after merge.
pub fn calculate_tokens_from_grid(&self, grid_t: usize, grid_h: usize, grid_w: usize) -> usize {
self.inner
.calculate_tokens_from_grid(grid_t, grid_h, grid_w)
}
/// Reshape pixel values from [C, H, W] to flattened patches format.
pub fn reshape_to_patches(
&self,
tensor: &Array3<f32>,
grid_t: usize,
grid_h: usize,
grid_w: usize,
) -> Vec<f32> {
self.inner
.reshape_to_patches(tensor, grid_t, grid_h, grid_w)
}
}
impl Deref for Qwen3VLProcessor {
type Target = QwenVLProcessorBase;
fn deref(&self) -> &Self::Target {
&self.inner
}
}
impl ImagePreProcessor for Qwen3VLProcessor {
fn default_mean(&self) -> [f64; 3] {
self.inner.default_mean()
}
fn default_std(&self) -> [f64; 3] {
self.inner.default_std()
}
fn preprocess(
&self,
images: &[DynamicImage],
config: &PreProcessorConfig,
) -> Result<PreprocessedImages, TransformError> {
self.inner.preprocess(images, config)
}
fn calculate_num_tokens(&self, width: u32, height: u32, config: &PreProcessorConfig) -> usize {
self.inner.calculate_num_tokens(width, height, config)
}
fn model_name(&self) -> &'static str {
self.inner.model_name()
}
fn get_processed_size(&self, config: &PreProcessorConfig) -> Option<(u32, u32)> {
self.inner.get_processed_size(config)
}
}
#[cfg(test)]
mod tests {
use image::{Rgb, RgbImage};
use super::*;
use crate::multimodal::vision::image_processor::ModelSpecificValue;
fn create_test_image(width: u32, height: u32, color: Rgb<u8>) -> DynamicImage {
DynamicImage::from(RgbImage::from_pixel(width, height, color))
}
#[test]
fn test_qwen3_vl_processor_default() {
let processor = Qwen3VLProcessor::new();
assert_eq!(processor.patch_size(), 16);
assert_eq!(processor.merge_size(), 2);
assert_eq!(processor.min_pixels(), DEFAULT_MIN_PIXELS);
assert_eq!(processor.max_pixels(), DEFAULT_MAX_PIXELS);
assert_eq!(processor.get_factor(), 32); // 16 * 2
}
#[test]
fn test_smart_resize_within_bounds() {
let processor = Qwen3VLProcessor::new();
// Image that's within bounds
let (h, w) = processor.smart_resize(500, 500).unwrap();
// Should be aligned to factor (32)
assert_eq!(h % 32, 0);
assert_eq!(w % 32, 0);
// Should be within bounds
assert!(h * w >= processor.min_pixels());
assert!(h * w <= processor.max_pixels());
}
#[test]
fn test_smart_resize_aspect_ratio_preserved() {
let processor = Qwen3VLProcessor::new();
// 2:1 aspect ratio
let (h, w) = processor.smart_resize(400, 800).unwrap();
// Aspect ratio should be approximately preserved
let original_ratio = 800.0 / 400.0;
let new_ratio = w as f64 / h as f64;
assert!((new_ratio - original_ratio).abs() < 0.5);
}
#[test]
fn test_smart_resize_extreme_aspect_ratio_error() {
let processor = Qwen3VLProcessor::new();
// 300:1 aspect ratio - should fail
let result = processor.smart_resize(100, 30000);
assert!(result.is_err());
}
#[test]
fn test_smart_resize_too_small_dimension_error() {
let processor = Qwen3VLProcessor::new();
// Dimension smaller than factor (32)
let result = processor.smart_resize(10, 100);
assert!(result.is_err());
}
#[test]
fn test_calculate_grid_thw_image() {
let processor = Qwen3VLProcessor::new();
// 480x640 image
let (t, h, w) = processor.calculate_grid_thw(480, 640, 1);
assert_eq!(t, 1); // Single image
assert_eq!(h, 480 / 16); // 30
assert_eq!(w, 640 / 16); // 40
}
#[test]
fn test_calculate_tokens() {
let processor = Qwen3VLProcessor::new();
// With merge_size=2, tokens = (t * h * w) / 4
let tokens = processor.calculate_tokens_from_grid(1, 30, 40);
assert_eq!(tokens, (30 * 40) / 4); // 300
}
#[test]
fn test_qwen3_vl_preprocess() {
let processor = Qwen3VLProcessor::new();
let config = PreProcessorConfig {
do_resize: Some(true),
do_normalize: Some(true),
image_mean: Some(QWEN3_MEAN.to_vec()),
image_std: Some(QWEN3_STD.to_vec()),
patch_size: Some(16),
merge_size: Some(2),
min_pixels: Some(DEFAULT_MIN_PIXELS),
max_pixels: Some(DEFAULT_MAX_PIXELS),
..Default::default()
};
let image = create_test_image(640, 480, Rgb([128, 128, 128]));
let result = processor.preprocess(&[image], &config).unwrap();
assert_eq!(result.batch_size(), 1);
// Check pixel values are normalized
let flat = result.pixel_values_flat();
// After normalization with [0.5, 0.5, 0.5] mean/std:
// (0.5 - 0.5) / 0.5 = 0.0 for gray
// Values should be in [-1, 1] range
assert!(flat.iter().all(|&v| (-1.5..=1.5).contains(&v)));
// Check image_grid_thw is present
assert!(result.model_specific.contains_key("image_grid_thw"));
// Verify token count is reasonable
assert!(result.num_img_tokens[0] > 0);
}
#[test]
fn test_qwen3_vl_preprocess_multiple() {
let processor = Qwen3VLProcessor::new();
let config = PreProcessorConfig {
image_mean: Some(QWEN3_MEAN.to_vec()),
image_std: Some(QWEN3_STD.to_vec()),
..Default::default()
};
let images = vec![
create_test_image(640, 480, Rgb([100, 100, 100])),
create_test_image(480, 640, Rgb([150, 150, 150])),
];
let result = processor.preprocess(&images, &config).unwrap();
// Both images processed
assert_eq!(result.image_sizes.len(), 2);
assert_eq!(result.num_img_tokens.len(), 2);
// Check grid_thw shape
if let Some(ModelSpecificValue::UintTensor { data, shape }) =
result.model_specific.get("image_grid_thw")
{
assert_eq!(shape, &[2, 3]); // 2 images, 3 values (T, H, W) each
assert_eq!(data.len(), 6);
} else {
panic!("Expected image_grid_thw to be UintTensor");
}
}
#[test]
fn test_qwen3_vl_from_config() {
let config = PreProcessorConfig {
patch_size: Some(16),
merge_size: Some(4),
min_pixels: Some(100000),
max_pixels: Some(500000),
temporal_patch_size: Some(4),
..Default::default()
};
let processor = Qwen3VLProcessor::from_preprocessor_config(&config);
assert_eq!(processor.patch_size(), 16);
assert_eq!(processor.merge_size(), 4);
assert_eq!(processor.min_pixels(), 100000);
assert_eq!(processor.max_pixels(), 500000);
assert_eq!(processor.temporal_patch_size(), 4);
}
#[test]
fn test_model_name() {
let processor = Qwen3VLProcessor::new();
assert_eq!(processor.model_name(), "qwen3-vl");
}
#[test]
fn test_default_mean_std() {
let processor = Qwen3VLProcessor::new();
assert_eq!(processor.default_mean(), QWEN3_MEAN);
assert_eq!(processor.default_std(), QWEN3_STD);
}
#[test]
fn test_qwen3_vs_qwen2_differences() {
// Verify the key differences from Qwen2-VL
let processor = Qwen3VLProcessor::new();
// Qwen3-VL uses patch_size=16 (vs 14 in Qwen2)
assert_eq!(processor.patch_size(), 16);
// Factor is 32 (vs 28 in Qwen2)
assert_eq!(processor.get_factor(), 32);
// Mean/std are [0.5, 0.5, 0.5] (vs CLIP values in Qwen2)
assert_eq!(processor.default_mean(), [0.5, 0.5, 0.5]);
assert_eq!(processor.default_std(), [0.5, 0.5, 0.5]);
}
}

475
sgl-router/src/multimodal/vision/processors/qwen_vl_base.rs Normal file
View File

@@ -0,0 +1,475 @@
//! Shared base implementation for Qwen VL family image processors.
//!
//! This module provides a generic processor that handles the common logic
//! for Qwen2-VL, Qwen2.5-VL, and Qwen3-VL models. The specific variants
//! differ only in their default parameters (patch_size, normalization values).
//!
//! # Processing Pipeline
//!
//! 1. Validate aspect ratio (must be < 200:1)
//! 2. Smart resize to fit within min/max pixel bounds
//! 3. Align dimensions to (patch_size * merge_size) boundary
//! 4. Convert to tensor and normalize
//! 5. Reshape into patches for the vision encoder
//!
//! # Token Calculation
//!
//! ```text
//! grid_t = 1 (for images, temporal dimension is 1)
//! grid_h = resized_height / patch_size
//! grid_w = resized_width / patch_size
//! num_tokens = (grid_t * grid_h * grid_w) / merge_size²
//! ```
use image::{DynamicImage, GenericImageView};
use ndarray::Array3;
use crate::multimodal::vision::{
image_processor::{ImagePreProcessor, ModelSpecificValue, PreprocessedImages},
preprocessor_config::PreProcessorConfig,
transforms::{normalize, pil_to_filter, resize, stack_batch, to_tensor, TransformError},
};
/// Configuration for a Qwen VL processor variant.
#[derive(Debug, Clone)]
pub struct QwenVLConfig {
/// Vision encoder patch size
pub patch_size: usize,
/// Merge size for token reduction
pub merge_size: usize,
/// Minimum total pixels allowed
pub min_pixels: usize,
/// Maximum total pixels allowed
pub max_pixels: usize,
/// Temporal patch size for video
pub temporal_patch_size: usize,
/// Normalization mean values
pub mean: [f64; 3],
/// Normalization std values
pub std: [f64; 3],
/// Model name for identification
pub model_name: &'static str,
}
/// Generic Qwen VL image processor.
///
/// This struct implements the shared preprocessing logic for all Qwen VL
/// model variants. Each variant (Qwen2-VL, Qwen3-VL, etc.) uses this with
/// different configuration values.
#[derive(Debug, Clone)]
pub struct QwenVLProcessorBase {
config: QwenVLConfig,
}
impl QwenVLProcessorBase {
/// Create a new processor with the given configuration.
pub fn new(config: QwenVLConfig) -> Self {
Self { config }
}
/// Get the patch size.
pub fn patch_size(&self) -> usize {
self.config.patch_size
}
/// Get the merge size.
pub fn merge_size(&self) -> usize {
self.config.merge_size
}
/// Get the minimum pixels.
pub fn min_pixels(&self) -> usize {
self.config.min_pixels
}
/// Get the maximum pixels.
pub fn max_pixels(&self) -> usize {
self.config.max_pixels
}
/// Get the temporal patch size.
pub fn temporal_patch_size(&self) -> usize {
self.config.temporal_patch_size
}
/// Get the factor for dimension alignment.
///
/// Dimensions must be divisible by (patch_size * merge_size).
#[inline]
pub fn get_factor(&self) -> usize {
self.config.patch_size * self.config.merge_size
}
/// Smart resize algorithm for Qwen VL models.
///
/// Resizes image dimensions to fit within min/max pixel bounds while:
/// - Preserving aspect ratio
/// - Aligning to (patch_size * merge_size) boundaries
///
/// # Arguments
/// * `height` - Original image height
/// * `width` - Original image width
///
/// # Returns
/// (new_height, new_width) or error if aspect ratio is too extreme
///
/// # Errors
/// - If height or width is smaller than the factor
/// - If aspect ratio exceeds 200:1
pub fn smart_resize(
&self,
height: usize,
width: usize,
) -> Result<(usize, usize), TransformError> {
let factor = self.get_factor();
// Validate minimum dimensions
if height < factor || width < factor {
return Err(TransformError::InvalidShape {
expected: format!("dimensions >= {} (patch_size * merge_size)", factor),
actual: vec![height, width],
});
}
// Validate aspect ratio
let max_dim = height.max(width) as f64;
let min_dim = height.min(width) as f64;
let aspect_ratio = max_dim / min_dim;
if aspect_ratio > 200.0 {
return Err(TransformError::InvalidShape {
expected: "aspect ratio < 200:1".to_string(),
actual: vec![height, width],
});
}
// Round to nearest factor multiple
let mut h_bar = (height as f64 / factor as f64).round() as usize * factor;
let mut w_bar = (width as f64 / factor as f64).round() as usize * factor;
// Ensure minimum size
h_bar = h_bar.max(factor);
w_bar = w_bar.max(factor);
// Scale down if exceeding max_pixels
if h_bar * w_bar > self.config.max_pixels {
let beta = ((height * width) as f64 / self.config.max_pixels as f64).sqrt();
h_bar = ((height as f64 / beta / factor as f64).floor() as usize) * factor;
w_bar = ((width as f64 / beta / factor as f64).floor() as usize) * factor;
// Ensure minimum size after scaling down
h_bar = h_bar.max(factor);
w_bar = w_bar.max(factor);
}
// Scale up if below min_pixels
else if h_bar * w_bar < self.config.min_pixels {
let beta = (self.config.min_pixels as f64 / (height * width) as f64).sqrt();
h_bar = ((height as f64 * beta / factor as f64).ceil() as usize) * factor;
w_bar = ((width as f64 * beta / factor as f64).ceil() as usize) * factor;
}
Ok((h_bar, w_bar))
}
/// Calculate the grid dimensions (T, H, W) for an image.
///
/// For single images, T=1. For video, T = num_frames / temporal_patch_size.
///
/// # Arguments
/// * `height` - Resized image height
/// * `width` - Resized image width
/// * `num_frames` - Number of frames (1 for images)
///
/// # Returns
/// (grid_t, grid_h, grid_w)
pub fn calculate_grid_thw(
&self,
height: usize,
width: usize,
num_frames: usize,
) -> (usize, usize, usize) {
let grid_t =
num_frames.max(self.config.temporal_patch_size) / self.config.temporal_patch_size;
let grid_h = height / self.config.patch_size;
let grid_w = width / self.config.patch_size;
(grid_t, grid_h, grid_w)
}
/// Calculate the number of image tokens after merge.
///
/// tokens = (grid_t * grid_h * grid_w) / merge_size²
pub fn calculate_tokens_from_grid(&self, grid_t: usize, grid_h: usize, grid_w: usize) -> usize {
(grid_t * grid_h * grid_w) / (self.config.merge_size * self.config.merge_size)
}
/// Reshape pixel values from [C, H, W] to flattened patches format.
///
/// This matches the HuggingFace Qwen2VLImageProcessor output format:
/// `(num_patches, patch_features)` where:
/// - num_patches = grid_t * grid_h * grid_w
/// - patch_features = C * temporal_patch_size * patch_size * patch_size
///
/// The transformation follows these steps (matching HuggingFace exactly):
/// 1. Start with [C, H, W] tensor, expand to [temporal, C, H, W]
/// 2. Reshape to [grid_t, temporal, C, grid_h/merge, merge, patch, grid_w/merge, merge, patch]
/// 3. Permute to [grid_t, grid_h/merge, grid_w/merge, merge, merge, C, temporal, patch, patch]
/// 4. Flatten to [num_patches, patch_features]
///
/// # Arguments
/// * `tensor` - Input tensor of shape [C, H, W]
/// * `grid_t` - Temporal grid size (1 for images)
/// * `grid_h` - Height grid size (H / patch_size)
/// * `grid_w` - Width grid size (W / patch_size)
///
/// # Returns
/// Flattened patches as Vec<f32> with shape semantics (num_patches, patch_features)
pub fn reshape_to_patches(
&self,
tensor: &Array3<f32>,
grid_t: usize,
grid_h: usize,
grid_w: usize,
) -> Vec<f32> {
use ndarray::IxDyn;
let channel = tensor.shape()[0];
let height = tensor.shape()[1];
let width = tensor.shape()[2];
let patch_size = self.config.patch_size;
let merge_size = self.config.merge_size;
let temporal_patch_size = self.config.temporal_patch_size;
// Verify dimensions match expected grid
debug_assert_eq!(
height,
grid_h * patch_size,
"Height must match grid_h * patch_size"
);
debug_assert_eq!(
width,
grid_w * patch_size,
"Width must match grid_w * patch_size"
);
// Step 1: Expand temporal dimension by replicating the frame
// [C, H, W] -> [temporal_patch_size, C, H, W]
let expanded = tensor
.view()
.insert_axis(ndarray::Axis(0))
.broadcast((temporal_patch_size, channel, height, width))
.expect("Broadcast failed")
.to_owned();
// Step 2: Reshape to split spatial dimensions into grid and patch components
// [temporal, C, H, W] -> [grid_t, temporal, C, grid_h/merge, merge, patch, grid_w/merge, merge, patch]
let grid_h_merged = grid_h / merge_size;
let grid_w_merged = grid_w / merge_size;
// Use IxDyn for 9-dimensional reshape (ndarray only supports up to Ix6 for fixed dims)
let shape_9d = IxDyn(&[
grid_t,
temporal_patch_size,
channel,
grid_h_merged,
merge_size,
patch_size,
grid_w_merged,
merge_size,
patch_size,
]);
let reshaped = expanded
.into_shape_with_order(shape_9d)
.expect("Reshape failed");
// Step 3: Permute axes to match HuggingFace output order
// From: [grid_t, temporal, C, grid_h/merge, merge, patch, grid_w/merge, merge, patch]
// [ 0 , 1 , 2, 3 , 4 , 5 , 6 , 7 , 8 ]
// To: [grid_t, grid_h/merge, grid_w/merge, merge, merge, C, temporal, patch, patch]
// [ 0 , 3 , 6 , 4 , 7 , 2, 1 , 5 , 8 ]
let permuted = reshaped.permuted_axes(&[0, 3, 6, 4, 7, 2, 1, 5, 8][..]);
// Step 4: Flatten to [num_patches, patch_features]
let num_patches = grid_t * grid_h * grid_w;
let patch_features = channel * temporal_patch_size * patch_size * patch_size;
// Make contiguous and flatten
let contiguous = permuted.as_standard_layout().into_owned();
let flat = contiguous
.into_shape_with_order(IxDyn(&[num_patches, patch_features]))
.expect("Final reshape failed");
let (vec, _offset) = flat.into_raw_vec_and_offset();
vec
}
}
impl ImagePreProcessor for QwenVLProcessorBase {
fn default_mean(&self) -> [f64; 3] {
self.config.mean
}
fn default_std(&self) -> [f64; 3] {
self.config.std
}
fn preprocess(
&self,
images: &[DynamicImage],
config: &PreProcessorConfig,
) -> Result<PreprocessedImages, TransformError> {
if images.is_empty() {
return Err(TransformError::EmptyBatch);
}
// Store original sizes
let image_sizes: Vec<(u32, u32)> = images.iter().map(|img| img.dimensions()).collect();
// First pass: calculate target dimensions for each image
let mut target_sizes = Vec::with_capacity(images.len());
for image in images {
let (w, h) = image.dimensions();
let (new_h, new_w) = self.smart_resize(h as usize, w as usize)?;
target_sizes.push((new_h, new_w));
}
// Find max height and width across all images
let max_height = target_sizes.iter().map(|(h, _)| *h).max().unwrap_or(0);
let max_width = target_sizes.iter().map(|(_, w)| *w).max().unwrap_or(0);
// Process each image with uniform max dimensions
let mean = config.get_image_mean();
let std = config.get_image_std();
let filter = pil_to_filter(config.resampling);
let mut tensors = Vec::with_capacity(images.len());
let mut grid_thw_data = Vec::with_capacity(images.len() * 3);
let mut num_img_tokens = Vec::with_capacity(images.len());
for (i, image) in images.iter().enumerate() {
let (target_h, target_w) = target_sizes[i];
// Resize to the target size for this image
let resized = if config.do_resize.unwrap_or(true) {
// For batching: resize to max dimensions to enable stacking
resize(image, max_width as u32, max_height as u32, filter)
} else {
image.clone()
};
// Convert to tensor
let mut tensor = to_tensor(&resized);
// Normalize
if config.do_normalize.unwrap_or(true) {
normalize(&mut tensor, &mean, &std);
}
tensors.push(tensor);
// Grid dimensions are based on the individual image's target size
let (grid_t, grid_h, grid_w) = self.calculate_grid_thw(target_h, target_w, 1);
grid_thw_data.push(grid_t as u32);
grid_thw_data.push(grid_h as u32);
grid_thw_data.push(grid_w as u32);
// Token count is based on individual grid
let tokens = self.calculate_tokens_from_grid(grid_t, grid_h, grid_w);
num_img_tokens.push(tokens);
}
// Stack tensors into batch (now all same size)
let pixel_values = stack_batch(&tensors)?;
// Create result with model-specific image_grid_thw
let result = PreprocessedImages::new(pixel_values, num_img_tokens, image_sizes).with_extra(
"image_grid_thw",
ModelSpecificValue::uint_2d(grid_thw_data, images.len(), 3),
);
Ok(result)
}
fn calculate_num_tokens(&self, width: u32, height: u32, _config: &PreProcessorConfig) -> usize {
// Calculate resized dimensions
let (new_height, new_width) = match self.smart_resize(height as usize, width as usize) {
Ok((h, w)) => (h, w),
Err(_) => {
// Fallback: use minimum size
let factor = self.get_factor();
(factor, factor)
}
};
// Calculate grid and tokens
let (grid_t, grid_h, grid_w) = self.calculate_grid_thw(new_height, new_width, 1);
self.calculate_tokens_from_grid(grid_t, grid_h, grid_w)
}
fn model_name(&self) -> &'static str {
self.config.model_name
}
fn get_processed_size(&self, _config: &PreProcessorConfig) -> Option<(u32, u32)> {
// Qwen VL models have dynamic sizing, no fixed output size
None
}
}
#[cfg(test)]
mod tests {
use super::*;
fn create_test_config() -> QwenVLConfig {
QwenVLConfig {
patch_size: 14,
merge_size: 2,
min_pixels: 256 * 28 * 28,
max_pixels: 1280 * 28 * 28,
temporal_patch_size: 2,
mean: [0.5, 0.5, 0.5],
std: [0.5, 0.5, 0.5],
model_name: "test-qwen-vl",
}
}
#[test]
fn test_qwen_vl_base_factor() {
let processor = QwenVLProcessorBase::new(create_test_config());
assert_eq!(processor.get_factor(), 28); // 14 * 2
}
#[test]
fn test_smart_resize_within_bounds() {
let processor = QwenVLProcessorBase::new(create_test_config());
let (h, w) = processor.smart_resize(500, 500).unwrap();
assert_eq!(h % 28, 0);
assert_eq!(w % 28, 0);
assert!(h * w >= processor.min_pixels());
assert!(h * w <= processor.max_pixels());
}
#[test]
fn test_smart_resize_extreme_aspect_ratio_error() {
let processor = QwenVLProcessorBase::new(create_test_config());
let result = processor.smart_resize(100, 30000);
assert!(result.is_err());
}
#[test]
fn test_calculate_grid_thw() {
let processor = QwenVLProcessorBase::new(create_test_config());
let (t, h, w) = processor.calculate_grid_thw(448, 448, 1);
assert_eq!(t, 1);
assert_eq!(h, 448 / 14);
assert_eq!(w, 448 / 14);
}
#[test]
fn test_calculate_tokens() {
let processor = QwenVLProcessorBase::new(create_test_config());
let tokens = processor.calculate_tokens_from_grid(1, 32, 32);
assert_eq!(tokens, (32 * 32) / 4);
}
}

152
sgl-router/tests/vision_golden_tests.rs
View File

@@ -7,6 +7,7 @@
//! - `llava/` - Standard CLIP processing (llava-hf/* models, no expand-to-square)
//! - `llava_pad/` - Expand-to-square mode (liuhaotian/llava-* models, image_aspect_ratio=pad)
//! - `qwen2_vl/` - Dynamic resolution with smart resize (Qwen/Qwen2-VL-* models)
//! - `qwen3_vl/` - Dynamic resolution with patch_size=16 and [0.5,0.5,0.5] norm (Qwen/Qwen3-VL-* models)
//!
//! To regenerate golden outputs:
//! ```bash
@@ -18,7 +19,7 @@ use std::{fs::File, io::Read, path::Path};
use ndarray::Array4;
use sgl_model_gateway::multimodal::vision::{
image_processor::ModelSpecificValue, ImagePreProcessor, LlavaProcessor, PreProcessorConfig,
Qwen2VLProcessor,
Qwen2VLProcessor, Qwen3VLProcessor,
};
/// Load a numpy .npz file and extract pixel_values
@@ -384,3 +385,152 @@ fn test_qwen2_vl_golden_wide() {
fn test_qwen2_vl_golden_small() {
run_qwen2_vl_golden_test("small");
}
// ============================================================================
// Qwen3-VL tests
// ============================================================================
/// Run a Qwen3-VL golden test for a specific image.
///
/// This test validates:
/// 1. image_grid_thw matches the HuggingFace output
/// 2. num_tokens calculation is correct
/// 3. Pixel values match after reshaping to patch format
///
/// Key differences from Qwen2-VL:
/// - patch_size: 16 (vs 14)
/// - factor: 32 (vs 28)
/// - normalization: [0.5, 0.5, 0.5] (vs CLIP)
fn run_qwen3_vl_golden_test(image_name: &str) {
let golden_dir = Path::new("tests/fixtures/golden/qwen3_vl");
let image_path = Path::new("tests/fixtures/images").join(format!("{}.jpg", image_name));
if !golden_dir.exists() || !image_path.exists() {
eprintln!(
"Golden test fixtures for qwen3_vl/{} not found, skipping test",
image_name
);
eprintln!("Run: python scripts/generate_vision_golden.py --model qwen3_vl");
return;
}
let npz_path = golden_dir.join(format!("golden_{}.npz", image_name));
let config = load_config(&golden_dir.join("preprocessor_config.json"));
// Load golden values
let golden_grid_thw = load_golden_grid_thw(&npz_path);
let golden_num_tokens = load_golden_num_tokens(&npz_path);
let (golden_pixels, golden_shape) = load_golden_qwen2_vl_pixels(&npz_path);
// Process image with our Rust processor
let image = image::open(&image_path).expect("Failed to open image");
let processor = Qwen3VLProcessor::from_preprocessor_config(&config);
let result = processor
.preprocess(&[image], &config)
.expect("Processing failed");
// Extract image_grid_thw from result
let rust_grid_thw = match result.model_specific.get("image_grid_thw") {
Some(ModelSpecificValue::UintTensor { data, shape }) => {
assert_eq!(shape, &[1, 3], "Expected shape [1, 3] for single image");
data.clone()
}
_ => panic!("Expected image_grid_thw in model_specific"),
};
// Compare grid dimensions
println!(
"qwen3_vl - {} image - Grid T H W: golden={:?}, rust={:?}",
image_name, golden_grid_thw, rust_grid_thw
);
assert_eq!(
golden_grid_thw, rust_grid_thw,
"image_grid_thw mismatch for {}",
image_name
);
// Compare token counts
let rust_num_tokens = result.num_img_tokens[0];
println!(
"qwen3_vl - {} image - Tokens: golden={}, rust={}",
image_name, golden_num_tokens, rust_num_tokens
);
assert_eq!(
golden_num_tokens, rust_num_tokens,
"num_tokens mismatch for {}",
image_name
);
// Compare pixel values by reshaping our output to patch format
let grid_t = rust_grid_thw[0] as usize;
let grid_h = rust_grid_thw[1] as usize;
let grid_w = rust_grid_thw[2] as usize;
// Get the tensor for the first image (batch index 0)
let pixel_values = &result.pixel_values;
let tensor_3d = pixel_values.index_axis(ndarray::Axis(0), 0).to_owned();
// Reshape to patches format
let rust_patches = processor.reshape_to_patches(&tensor_3d, grid_t, grid_h, grid_w);
// Verify shapes match (Qwen3-VL has patch_size=16)
let expected_num_patches = grid_t * grid_h * grid_w;
let patch_size = config.patch_size.unwrap_or(16);
let temporal_patch_size = config.temporal_patch_size.unwrap_or(2);
let expected_patch_features = 3 * temporal_patch_size * patch_size * patch_size;
println!(
"qwen3_vl - {} image - Patch shape: golden={:?}, rust=({}, {})",
image_name, golden_shape, expected_num_patches, expected_patch_features
);
assert_eq!(
golden_shape,
(expected_num_patches, expected_patch_features),
"Patch shape mismatch"
);
assert_eq!(
rust_patches.len(),
expected_num_patches * expected_patch_features,
"Rust patches size mismatch"
);
// Compare pixel values
let max_diff = rust_patches
.iter()
.zip(golden_pixels.iter())
.map(|(r, g)| (r - g).abs())
.fold(0.0f32, f32::max);
println!(
"qwen3_vl - {} image - Max pixel diff: {:.6}",
image_name, max_diff
);
// Allow tolerance for floating point and interpolation differences
assert!(
max_diff < 0.02,
"Max pixel difference {} exceeds tolerance 0.02 for {}",
max_diff,
image_name
);
}
#[test]
fn test_qwen3_vl_golden_square() {
run_qwen3_vl_golden_test("square");
}
#[test]
fn test_qwen3_vl_golden_tall() {
run_qwen3_vl_golden_test("tall");
}
#[test]
fn test_qwen3_vl_golden_wide() {
run_qwen3_vl_golden_test("wide");
}
#[test]
fn test_qwen3_vl_golden_small() {
run_qwen3_vl_golden_test("small");
}