[model-gateway] add qwen3_vl model image processor (#14377)

This commit is contained in:
Simo Lin
2025-12-03 17:16:37 -08:00
committed by GitHub
parent 388151053d
commit d42c167bfd
8 changed files with 1272 additions and 346 deletions
+151 -1
View File
@@ -7,6 +7,7 @@
//! - `llava/` - Standard CLIP processing (llava-hf/* models, no expand-to-square)
//! - `llava_pad/` - Expand-to-square mode (liuhaotian/llava-* models, image_aspect_ratio=pad)
//! - `qwen2_vl/` - Dynamic resolution with smart resize (Qwen/Qwen2-VL-* models)
//! - `qwen3_vl/` - Dynamic resolution with patch_size=16 and [0.5,0.5,0.5] norm (Qwen/Qwen3-VL-* models)
//!
//! To regenerate golden outputs:
//! ```bash
@@ -18,7 +19,7 @@ use std::{fs::File, io::Read, path::Path};
use ndarray::Array4;
use sgl_model_gateway::multimodal::vision::{
image_processor::ModelSpecificValue, ImagePreProcessor, LlavaProcessor, PreProcessorConfig,
Qwen2VLProcessor,
Qwen2VLProcessor, Qwen3VLProcessor,
};
/// Load a numpy .npz file and extract pixel_values
@@ -384,3 +385,152 @@ fn test_qwen2_vl_golden_wide() {
fn test_qwen2_vl_golden_small() {
run_qwen2_vl_golden_test("small");
}
// ============================================================================
// Qwen3-VL tests
// ============================================================================
/// Run a Qwen3-VL golden test for a specific image.
///
/// This test validates:
/// 1. image_grid_thw matches the HuggingFace output
/// 2. num_tokens calculation is correct
/// 3. Pixel values match after reshaping to patch format
///
/// Key differences from Qwen2-VL:
/// - patch_size: 16 (vs 14)
/// - factor: 32 (vs 28)
/// - normalization: [0.5, 0.5, 0.5] (vs CLIP)
fn run_qwen3_vl_golden_test(image_name: &str) {
let golden_dir = Path::new("tests/fixtures/golden/qwen3_vl");
let image_path = Path::new("tests/fixtures/images").join(format!("{}.jpg", image_name));
if !golden_dir.exists() || !image_path.exists() {
eprintln!(
"Golden test fixtures for qwen3_vl/{} not found, skipping test",
image_name
);
eprintln!("Run: python scripts/generate_vision_golden.py --model qwen3_vl");
return;
}
let npz_path = golden_dir.join(format!("golden_{}.npz", image_name));
let config = load_config(&golden_dir.join("preprocessor_config.json"));
// Load golden values
let golden_grid_thw = load_golden_grid_thw(&npz_path);
let golden_num_tokens = load_golden_num_tokens(&npz_path);
let (golden_pixels, golden_shape) = load_golden_qwen2_vl_pixels(&npz_path);
// Process image with our Rust processor
let image = image::open(&image_path).expect("Failed to open image");
let processor = Qwen3VLProcessor::from_preprocessor_config(&config);
let result = processor
.preprocess(&[image], &config)
.expect("Processing failed");
// Extract image_grid_thw from result
let rust_grid_thw = match result.model_specific.get("image_grid_thw") {
Some(ModelSpecificValue::UintTensor { data, shape }) => {
assert_eq!(shape, &[1, 3], "Expected shape [1, 3] for single image");
data.clone()
}
_ => panic!("Expected image_grid_thw in model_specific"),
};
// Compare grid dimensions
println!(
"qwen3_vl - {} image - Grid T H W: golden={:?}, rust={:?}",
image_name, golden_grid_thw, rust_grid_thw
);
assert_eq!(
golden_grid_thw, rust_grid_thw,
"image_grid_thw mismatch for {}",
image_name
);
// Compare token counts
let rust_num_tokens = result.num_img_tokens[0];
println!(
"qwen3_vl - {} image - Tokens: golden={}, rust={}",
image_name, golden_num_tokens, rust_num_tokens
);
assert_eq!(
golden_num_tokens, rust_num_tokens,
"num_tokens mismatch for {}",
image_name
);
// Compare pixel values by reshaping our output to patch format
let grid_t = rust_grid_thw[0] as usize;
let grid_h = rust_grid_thw[1] as usize;
let grid_w = rust_grid_thw[2] as usize;
// Get the tensor for the first image (batch index 0)
let pixel_values = &result.pixel_values;
let tensor_3d = pixel_values.index_axis(ndarray::Axis(0), 0).to_owned();
// Reshape to patches format
let rust_patches = processor.reshape_to_patches(&tensor_3d, grid_t, grid_h, grid_w);
// Verify shapes match (Qwen3-VL has patch_size=16)
let expected_num_patches = grid_t * grid_h * grid_w;
let patch_size = config.patch_size.unwrap_or(16);
let temporal_patch_size = config.temporal_patch_size.unwrap_or(2);
let expected_patch_features = 3 * temporal_patch_size * patch_size * patch_size;
println!(
"qwen3_vl - {} image - Patch shape: golden={:?}, rust=({}, {})",
image_name, golden_shape, expected_num_patches, expected_patch_features
);
assert_eq!(
golden_shape,
(expected_num_patches, expected_patch_features),
"Patch shape mismatch"
);
assert_eq!(
rust_patches.len(),
expected_num_patches * expected_patch_features,
"Rust patches size mismatch"
);
// Compare pixel values
let max_diff = rust_patches
.iter()
.zip(golden_pixels.iter())
.map(|(r, g)| (r - g).abs())
.fold(0.0f32, f32::max);
println!(
"qwen3_vl - {} image - Max pixel diff: {:.6}",
image_name, max_diff
);
// Allow tolerance for floating point and interpolation differences
assert!(
max_diff < 0.02,
"Max pixel difference {} exceeds tolerance 0.02 for {}",
max_diff,
image_name
);
}
#[test]
fn test_qwen3_vl_golden_square() {
run_qwen3_vl_golden_test("square");
}
#[test]
fn test_qwen3_vl_golden_tall() {
run_qwen3_vl_golden_test("tall");
}
#[test]
fn test_qwen3_vl_golden_wide() {
run_qwen3_vl_golden_test("wide");
}
#[test]
fn test_qwen3_vl_golden_small() {
run_qwen3_vl_golden_test("small");
}