vlm: refactor engine vlm params and support processor output as input (#14091)

Co-authored-by: Mick <mickjagger19@icloud.com>
Co-authored-by: zhaochenyang20 <zhaochenyang20@gmail.com>
Co-authored-by: Xinyuan Tong <115166877+JustinTong0323@users.noreply.github.com>
Co-authored-by: BenYao21 <cyao22@asu.edu>
Co-authored-by: minleminzui <minleminzui@gmail.com>
Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Co-authored-by: 赵晨阳 <zhaochen20@outlook.com>

python/sglang/srt/entrypoints/engine.py

@@ -273,6 +273,8 @@ class Engine(EngineBase):
         # - Single image for a single request
         # - List of images (one per request in a batch)
         # - List of lists of images (multiple images per request)
+        # - List of preprocessed outputs from a Huggingface processor, each as a dict containing `format`: 'processor_output' and other data
+        # - List of precomputed image embeddings, each as a dict containing field `format`: 'precomputed_embedding' and `feature`: the precomputed embedding
         # See also python/sglang/srt/utils.py:load_image for more details.
         image_data: Optional[MultimodalDataInputFormat] = None,
         audio_data: Optional[MultimodalDataInputFormat] = None,
@@ -355,6 +357,8 @@ class Engine(EngineBase):
         # - Single image for a single request
         # - List of images (one per request in a batch)
         # - List of lists of images (multiple images per request)
+        # - List of preprocessed outputs from a Huggingface processor, each as a dict containing `format`: 'processor_output' and other data
+        # - List of precomputed image embeddings, each as a dict containing field `format`: 'precomputed_embedding' and `feature`: the precomputed embedding
         # See also python/sglang/srt/utils.py:load_image for more details.
         image_data: Optional[MultimodalDataInputFormat] = None,
         audio_data: Optional[MultimodalDataInputFormat] = None,

python/sglang/srt/function_call/llama32_detector.py

@@ -1,5 +1,7 @@
+import ast
 import json
 import logging
+import re
 from typing import List
 
 from sglang.srt.entrypoints.openai.protocol import Tool
@@ -32,6 +34,16 @@ class Llama32Detector(BaseFormatDetector):
         # if users define to use a different separator in their prompt
         self.tool_call_separator = ";"
 
+    def _convert_python_dict_to_json(self, text: str) -> str:
+        """Convert Python dict strings to JSON format."""
+        try:
+            parsed = ast.literal_eval(text.strip())
+            if isinstance(parsed, dict):
+                return json.dumps(parsed, ensure_ascii=False)
+        except:
+            pass
+        return text
+
     def has_tool_call(self, text: str) -> bool:
         """Check if the text contains a Llama 3.2 format tool call."""
         # depending on the prompt format the Llama model may or may not
@@ -59,16 +71,36 @@ class Llama32Detector(BaseFormatDetector):
                 all_actions.append(obj)
                 idx += end + len(self.tool_call_separator)
                 safe_idx = idx
-            except json.JSONDecodeError as e:
-                # Find where next `{"name"` appears and try again
-                logger.warning(
-                    f"Failed to parse JSON part: {action_text[idx:]}, JSON parse error: {str(e)}"
-                )
+            except json.JSONDecodeError:
+                # Try Python dict conversion as fallback
+                try:
+                    dict_end = idx
+                    brace_count = 0
+                    for i in range(idx, action_text_len):
+                        if action_text[i] == "{":
+                            brace_count += 1
+                        elif action_text[i] == "}":
+                            brace_count -= 1
+                            if brace_count == 0:
+                                dict_end = i + 1
+                                break
+
+                    if dict_end > idx:
+                        potential_dict = action_text[idx:dict_end]
+                        json_version = self._convert_python_dict_to_json(potential_dict)
+                        if json_version != potential_dict:
+                            obj, _ = decoder.raw_decode(json_version)
+                            all_actions.append(obj)
+                            idx = dict_end + len(self.tool_call_separator)
+                            safe_idx = idx
+                            continue
+                except:
+                    pass
+
                 next_obj_start = action_text.find('{"name":', idx + 1)
                 if next_obj_start == -1:
                     break
                 idx = next_obj_start
-                continue
 
         # Only process if we found valid JSON objects
         calls = self.parse_base_json(all_actions, tools) if all_actions else []
@@ -80,6 +112,30 @@ class Llama32Detector(BaseFormatDetector):
             normal_text=normal_text + trailing_text, calls=calls
         )
 
+    def parse_streaming_increment(
+        self, new_text: str, tools: List[Tool]
+    ) -> StreamingParseResult:
+        """Override to handle Python dict format in streaming."""
+        # First try with converted Python dict
+        self._buffer += new_text
+        converted_buffer = self._buffer
+
+        # Convert Python dict syntax to JSON
+        converted_buffer = re.sub(r"'([^']*)':", r'"\1":', converted_buffer)
+        converted_buffer = re.sub(r":\s*'([^']*)'", r': "\1"', converted_buffer)
+
+        # Temporarily replace buffer for parsing
+        original_buffer = self._buffer
+        self._buffer = converted_buffer
+
+        try:
+            result = super().parse_streaming_increment("", tools)
+            return result
+        except:
+            # Fall back to original buffer
+            self._buffer = original_buffer
+            return super().parse_streaming_increment(new_text, tools)
+
     def structure_info(self) -> _GetInfoFunc:
         return lambda name: StructureInfo(
             begin='<|python_tag|>{"name":"' + name + '", "arguments":',

python/sglang/srt/managers/schedule_batch.py

@@ -189,6 +189,12 @@ class Modality(Enum):
         return [Modality.IMAGE, Modality.VIDEO, Modality.AUDIO]
 
 
+class MultimodalInputFormat(Enum):
+    NORMAL = auto()
+    PROCESSOR_OUTPUT = auto()
+    PRECOMPUTED_EMBEDDING = auto()
+
+
 @dataclasses.dataclass
 class MultimodalDataItem:
     """
@@ -204,6 +210,8 @@ class MultimodalDataItem:
     pad_value: int = None
     offsets: Optional[list] = None
 
+    format: MultimodalInputFormat = MultimodalInputFormat.NORMAL
+
     # the raw features returned by processor, e.g. pixel_values or audio_features
     feature: Union[torch.Tensor, np.ndarray] = None
     # the precomputed embeddings, passed as final encoder embeddings
@@ -276,6 +284,9 @@ class MultimodalDataItem:
         ...
         # TODO
 
+    def is_precomputed_embedding(self):
+        return self.format == MultimodalInputFormat.PRECOMPUTED_EMBEDDING
+
     @staticmethod
     def from_dict(obj: dict):
         kwargs = dict(obj)

python/sglang/srt/models/gemma3_causal.py

@@ -373,14 +373,20 @@ class Gemma3RotaryEmbedding(nn.Module):
         # BC: "rope_type" was originally "type"
         if hasattr(config, "rope_scaling") and config.rope_scaling is not None:
             self.rope_type = config.rope_scaling.get(
-                "rope_type", config.rope_scaling.get("type")
+                "rope_type", config.rope_scaling.get("type", "default")
             )
+
         else:
             self.rope_type = "default"
+
+        if self.rope_type is None:
+            self.rope_type = "default"
+
         self.max_seq_len_cached = config.max_position_embeddings
         self.original_max_seq_len = config.max_position_embeddings
 
         self.config = config
+
         self.rope_init_fn = ROPE_INIT_FUNCTIONS[self.rope_type]
 
         inv_freq, self.attention_scaling = self.rope_init_fn(self.config, device)

python/sglang/srt/models/gemma3_mm.py

@@ -290,15 +290,26 @@ class Gemma3ForConditionalGeneration(PreTrainedModel):
     def get_image_feature(self, items: List[MultimodalDataItem]):
         """
         Projects the last hidden state from the vision model into language model space.
+        Supports both raw image pixel values and precomputed embeddings.
 
         Returns:
             image_features (`torch.Tensor`): Image feature tensor of shape `(num_images, image_length, embed_dim)`).
         """
         # Process images one by one to handle flatten_batch=True constraint in vision_tower
         all_pixel_values = flatten_nested_list([item.feature for item in items])
-        vision_outputs_list = []
+
+        final_features_list = []
 
         for pixel_values_batch in all_pixel_values:
+            if (
+                pixel_values_batch.dim() == 3
+                and pixel_values_batch.shape[-1] == self.config.text_config.hidden_size
+            ):
+                final_features_list.append(
+                    pixel_values_batch.to(self.language_model.device)
+                )
+                continue
+
             # Normalize input shape to [batch_size, channels, height, width]
             if pixel_values_batch.dim() == 5:
                 pixel_values_batch = pixel_values_batch.squeeze(0)
@@ -309,20 +320,29 @@ class Gemma3ForConditionalGeneration(PreTrainedModel):
                     f"Unexpected pixel_values shape: {pixel_values_batch.shape}"
                 )
 
-            # Process each image in the batch
+            # Process each image in the batch through Vision Tower
+            batch_vision_outputs = []
             batch_size = pixel_values_batch.shape[0]
+
             for i in range(batch_size):
                 pixel_value = pixel_values_batch[i : i + 1]  # Keep batch dimension as 1
                 pixel_value = pixel_value.to(
                     device=self.vision_tower.device, dtype=self.language_model.dtype()
                 )
                 vision_output = self.vision_tower(pixel_values=pixel_value)
-                vision_outputs_list.append(vision_output)
+                batch_vision_outputs.append(vision_output)
 
-        # Concatenate all vision outputs
-        vision_outputs = torch.cat(vision_outputs_list, dim=0)
-        image_features = self.multi_modal_projector(vision_outputs)
-        return image_features
+            if batch_vision_outputs:
+                vision_outputs_cat = torch.cat(batch_vision_outputs, dim=0)
+
+                projected_features = self.multi_modal_projector(vision_outputs_cat)
+                final_features_list.append(projected_features)
+
+        # Concatenate all features (all are now in text space)
+        if final_features_list:
+            return torch.cat(final_features_list, dim=0)
+        else:
+            return torch.tensor([], device=self.language_model.device)
 
     @torch.no_grad()
     def forward(

python/sglang/srt/models/kimi_vl.py

@@ -142,6 +142,13 @@ class KimiVLForConditionalGeneration(nn.Module):
             .type(self.vision_tower.dtype)
             .to(self.vision_tower.device)
         )
+
+        if (
+            pixel_values.dim() == 2
+            and pixel_values.shape[-1] == self.config.text_config.hidden_size
+        ):
+            return pixel_values
+
         image_grid_hws = torch.cat([item.image_grid_hws for item in items], dim=0).to(
             self.vision_tower.device
         )

python/sglang/srt/models/minicpmo.py

@@ -1668,6 +1668,24 @@ class MiniCPMO(MiniCPMBaseModel):
             [item.audio_feature_lens for item in items if item.audio_feature_lens]
         )
 
+        # Ensure audio_feature_lens_raw is properly formatted as [[tensor], [tensor], ...]
+        if audio_feature_lens_raw:
+            if isinstance(audio_feature_lens_raw[0], torch.Tensor):
+                # Flat list of tensors, wrap each in a list
+                audio_feature_lens_raw = [[lens] for lens in audio_feature_lens_raw]
+            elif isinstance(audio_feature_lens_raw[0], list):
+                # Already nested, ensure all elements are properly formatted
+                # Flatten if needed
+                flattened = []
+                for item in audio_feature_lens_raw:
+                    if isinstance(item, list):
+                        flattened.extend(item)
+                    else:
+                        flattened.append(item)
+                audio_feature_lens_raw = [
+                    [item] if not isinstance(item, list) else item for item in flattened
+                ]
+
         final_audio_embeds = []
 
         assert isinstance(wavforms, list)
@@ -1675,7 +1693,14 @@ class MiniCPMO(MiniCPMBaseModel):
         # exist audio
         for wavform in wavforms:
             if len(wavform) > 0:
-                audio_feature_lens = torch.hstack(audio_feature_lens_raw)
+                # Flatten audio_feature_lens_raw to get a list of tensors
+                flattened_lens = []
+                for item in audio_feature_lens_raw:
+                    if isinstance(item, list):
+                        flattened_lens.extend(item)
+                    else:
+                        flattened_lens.append(item)
+                audio_feature_lens = torch.hstack(flattened_lens)
                 batch_size, _, max_mel_seq_len = wavform.shape
                 max_seq_len = (max_mel_seq_len - 1) // 2 + 1
 

python/sglang/srt/models/qwen2_5_vl.py

@@ -447,7 +447,10 @@ class Qwen2_5_VisionTransformer(nn.Module, RotaryPosMixin):
         # transformers
         x = x.unsqueeze(1)
         for layer_num, blk in enumerate(self.blocks):
-            if layer_num in self.fullatt_block_indexes:
+            fullatt_indexes = self.fullatt_block_indexes
+            if isinstance(fullatt_indexes, torch.Tensor):
+                fullatt_indexes = fullatt_indexes.tolist()
+            if layer_num in fullatt_indexes:
                 cu_seqlens_now = cu_seqlens
             else:
                 cu_seqlens_now = cu_window_seqlens
@@ -630,6 +633,25 @@ class Qwen2_5_VLForConditionalGeneration(nn.Module):
             self.visual.dtype
         )
         image_grid_thw = torch.concat([item.image_grid_thw for item in items], dim=0)
+
+        expected_dim = getattr(self.visual, "embed_dim", -1)
+
+        if expected_dim == -1:
+            vision_conf = self.config.vision_config
+            expected_dim = getattr(
+                vision_conf, "embed_dim", getattr(vision_conf, "hidden_size", -1)
+            )
+
+        raw_patch_dim = 1176
+
+        if pixel_values.dim() == 2:
+            current_dim = pixel_values.shape[-1]
+            if current_dim == expected_dim:
+                return pixel_values
+            if current_dim != raw_patch_dim:
+
+                return pixel_values
+
         assert pixel_values.dim() == 2, pixel_values.dim()
         assert image_grid_thw.dim() == 2, image_grid_thw.dim()
         if self.use_data_parallel:

python/sglang/srt/multimodal/processors/base_processor.py

@@ -12,9 +12,12 @@ import torch
 from PIL import Image
 from transformers import BaseImageProcessorFast
 
-from sglang.srt.environ import envs
-from sglang.srt.managers.schedule_batch import Modality, MultimodalDataItem
-from sglang.srt.utils import is_npu, load_audio, load_image, load_video, logger
+from sglang.srt.managers.schedule_batch import (
+    Modality,
+    MultimodalDataItem,
+    MultimodalInputFormat,
+)
+from sglang.srt.utils import envs, is_npu, load_audio, load_image, load_video, logger
 from sglang.srt.utils.cuda_ipc_transport_utils import (
     MM_FEATURE_CACHE_SIZE,
     MM_ITEM_MEMORY_POOL_RECYCLE_INTERVAL,
@@ -29,7 +32,7 @@ SGL_USE_CUDA_IPC = envs.SGLANG_USE_CUDA_IPC_TRANSPORT.get()
 
 @dataclasses.dataclass
 class BaseMultiModalProcessorOutput:
-    # input_text, with each frame of video/image represented with a image_token
+    # input_text with all multimodality placeholder token expanded
     input_text: str
 
     # frames loaded from image, in given order
@@ -385,11 +388,18 @@ class BaseMultimodalProcessor(ABC):
         """
         Load a single multimodal data.
 
-        If data is precomputed, returns directly.
+        If data is processor_output or precomputed embedding, return directly.
 
         Static method that can be pickled for multiprocessing"""
         if isinstance(data, dict):
-            return data
+            data_format = data.get("format")
+            if data_format in (
+                MultimodalInputFormat.PROCESSOR_OUTPUT.name,
+                MultimodalInputFormat.PRECOMPUTED_EMBEDDING.name,
+                "processor_output",
+                "precomputed_embedding",
+            ):
+                return data
         try:
             if modality == Modality.IMAGE:
                 img, _ = load_image(data)
@@ -431,9 +441,10 @@ class BaseMultimodalProcessor(ABC):
                 try:
                     data = next(data_iterator)
                 except StopIteration:
-                    raise ValueError(
-                        f"Mismatch: More '{text_part}' tokens found than corresponding data items provided."
+                    logger.warning(
+                        f"Mismatch: More '{modality.name}' tokens found than corresponding data provided."
                     )
+                    return futures, task_info
 
                 frame_count_limit = None
                 if modality == Modality.IMAGE and image_estimated_frames_iter:
@@ -475,6 +486,77 @@ class BaseMultimodalProcessor(ABC):
 
         return futures, task_info
 
+    @staticmethod
+    def _validate_one_modality(modality: Modality, data_list: Optional[list]):
+        if data_list is None:
+            return
+        if not isinstance(data_list, list):
+            raise TypeError(
+                f"{modality.name} must be a list or None, got {type(data_list)}"
+            )
+
+        formatted_indices = []
+        for idx, item in enumerate(data_list):
+            if isinstance(item, dict):
+                fmt = item.get("format")
+                if fmt in {"processor_output", "precomputed_embedding"}:
+                    formatted_indices.append(idx)
+
+        if formatted_indices:
+            if len(data_list) != 1:
+                raise ValueError(
+                    f"For {modality}, when providing a 'processor_output' or "
+                    f"'precomputed_embedding', you must pass exactly one item; "
+                    f"received {len(data_list)} items (formatted at indices {formatted_indices})."
+                )
+
+    @staticmethod
+    def validate_mm_data(
+        image_data: Optional[list] = None,
+        video_data: Optional[list] = None,
+        audio_data: Optional[list] = None,
+    ):
+        """
+        Validate multimodal input lists per modality.
+
+        Rule per modality (image/video/audio):
+        - Either the list has exactly one item and that single item is a dict with
+          format in {"processor_output", "precomputed_embedding"};
+        - Or, the list contains only "normal" items (i.e., does not include any
+          item whose format is one of the two above).
+
+        Empty or None lists are considered valid.
+        """
+
+        BaseMultimodalProcessor._validate_one_modality(Modality.IMAGE, image_data)
+        BaseMultimodalProcessor._validate_one_modality(Modality.VIDEO, video_data)
+        BaseMultimodalProcessor._validate_one_modality(Modality.AUDIO, audio_data)
+
+    def _process_loaded_mm_data(self, modality, raw_data, result):
+        images, videos, audios = [], [], []
+
+        is_precomputed = isinstance(raw_data, dict) and raw_data.get("format") in [
+            MultimodalInputFormat.PROCESSOR_OUTPUT.name,
+            MultimodalInputFormat.PRECOMPUTED_EMBEDDING.name,
+            "processor_output",
+            "precomputed_embedding",
+        ]
+
+        if modality == Modality.IMAGE:
+            if is_precomputed:
+                images.append(result)
+            else:
+                if isinstance(result, list):
+                    images.extend(result)
+                else:
+                    images.append(result)
+        elif modality == Modality.VIDEO:
+            videos.append(result)
+        elif modality == Modality.AUDIO:
+            audios.append(result)
+
+        return is_precomputed, images, videos, audios
+
     def load_mm_data(
         self,
         prompt: str,
@@ -495,8 +577,10 @@ class BaseMultimodalProcessor(ABC):
             discard_alpha_channel: if True, discards the alpha channel in the returned images
 
         """
-        multimodal_tokens_pattern = multimodal_tokens.get_combined_regex()
 
+        BaseMultimodalProcessor.validate_mm_data(image_data, video_data, audio_data)
+
+        multimodal_tokens_pattern = multimodal_tokens.get_combined_regex()
         if isinstance(prompt, list) and return_text:
             assert len(prompt) and isinstance(prompt[0], int)
             prompt = self._processor.tokenizer.decode(prompt)
@@ -506,7 +590,6 @@ class BaseMultimodalProcessor(ABC):
         assert isinstance(prompt, str)
         # split text into list of normal text and special tokens
         text_parts = re.split(multimodal_tokens_pattern, prompt)
-
         # collect all data
         data_iterators = {}
         if multimodal_tokens.image_token and image_data:
@@ -531,29 +614,31 @@ class BaseMultimodalProcessor(ABC):
         # Process results
         images, videos, audios = [], [], []
         new_text_parts = []
+        has_precomputed_input = False
         for text_part in text_parts:
             try:
                 if multimodal_tokens_pattern.match(text_part):
                     modality, raw_data, frame_limit = next(task_info_iter)
-                    is_precomputed = isinstance(raw_data, dict)
                     result = next(futures_iter).result()
 
+                    is_precomputed, new_imgs, new_vids, new_auds = (
+                        self._process_loaded_mm_data(modality, raw_data, result)
+                    )
+
+                    has_precomputed_input |= is_precomputed
+                    images.extend(new_imgs)
+                    videos.extend(new_vids)
+                    audios.extend(new_auds)
+
                     if modality == Modality.IMAGE:
-                        # If data is already processed it will be a
-                        # dictionary(precomputed). In this case we want to keep the
-                        # expanded tokens in text_part. Otherwise, we will
-                        # call the processor code, so keep only a single image
-                        # token.
-                        mm_tokens = (
-                            text_part
-                            if is_precomputed
-                            else multimodal_tokens.image_token
-                        )
-                        frames = [result] if not isinstance(result, list) else result
-                        if frames:
-                            # only for minicpmv
-                            images += frames
-                            new_text_parts += mm_tokens * len(frames)
+                        if is_precomputed:
+                            new_text_parts += [text_part]
+                        else:
+                            count = len(new_imgs)
+                            if count > 0:
+                                new_text_parts += [
+                                    multimodal_tokens.image_token
+                                ] * count
                     elif modality == Modality.VIDEO:
                         # load as video
                         mm_tokens = (
@@ -561,7 +646,6 @@ class BaseMultimodalProcessor(ABC):
                             if is_precomputed
                             else multimodal_tokens.video_token
                         )
-                        videos += [result]
                         new_text_parts += mm_tokens
                     elif modality == Modality.AUDIO:
                         # audio
@@ -570,12 +654,19 @@ class BaseMultimodalProcessor(ABC):
                             if is_precomputed
                             else multimodal_tokens.audio_token
                         )
-                        audios += [result]
                         new_text_parts += mm_tokens
                 else:
                     # normal text
                     new_text_parts += [text_part]
 
+            except StopIteration as e:
+                # when precomputed_input is presented with multi-images, StopIteration is expected
+                if has_precomputed_input:
+                    new_text_parts += [text_part]
+                    continue
+                raise RuntimeError(
+                    f"An exception occurred while loading multimodal data: {e}"
+                )
             except Exception as e:
                 raise RuntimeError(
                     f"An exception occurred while loading multimodal data: {e}"
@@ -601,7 +692,6 @@ class BaseMultimodalProcessor(ABC):
         mask = input_ids == mm_token_id
         start_positions = (mask & ~torch.roll(mask, 1)).nonzero(as_tuple=True)[0]
         end_positions = (mask & ~torch.roll(mask, -1)).nonzero(as_tuple=True)[0]
-
         return list(zip(start_positions.tolist(), end_positions.tolist()))
 
     @staticmethod
@@ -614,35 +704,42 @@ class BaseMultimodalProcessor(ABC):
         return list(zip(indices_start.tolist(), indices_end.tolist()))
 
     def collect_mm_items_from_processor_output(
-        self, data_dict: dict
+        self, data_dict: dict, modality: Modality = None
     ) -> List[MultimodalDataItem]:
-        """Create mm_items directly from processor output."""
+        """
+        Create mm_items directly from processor output, with one item for each modality
+
+        Note that the data_dict can be passed via offline engine api
+        """
+
         items: dict[Modality, MultimodalDataItem] = {}
         for attr_name, value in data_dict.items():
             if attr_name == "input_ids":
                 continue
 
             # Get modality for this attribute
-            modality = self.ATTR_NAME_TO_MODALITY.get(attr_name)
+            current_modality = modality or self.ATTR_NAME_TO_MODALITY.get(attr_name)
 
             if attr_name == "precomputed_embeddings":
                 modality_str = data_dict.get("modality")
-                modality = Modality.IMAGE
+                current_modality = Modality.IMAGE
                 if modality_str:
                     try:
-                        modality = Modality.from_str(modality_str)
+                        current_modality = Modality.from_str(modality_str)
                     except ValueError:
                         pass
 
-            if modality:
+            if current_modality:
                 # Create item if needed
-                if modality not in items:
-                    items[modality] = MultimodalDataItem(modality=modality)
+                if current_modality not in items:
+                    items[current_modality] = MultimodalDataItem(
+                        modality=current_modality
+                    )
 
                 if attr_name in self.FEATURE_NAMES:
                     attr_name = "feature"
 
-                items[modality].set(attr_name, value)
+                items[current_modality].set(attr_name, value)
 
         return list(items.values())
 
@@ -678,9 +775,9 @@ class BaseMultimodalProcessor(ABC):
             Tuple of (list of mm_items, input_ids)
         """
         # Collect all items and categorize them
-        all_items = base_output.organize_results()
+        all_loaded_data = base_output.organize_results()
         # Handle text-only case
-        if not all_items:
+        if not all_loaded_data:
             input_ids = self._processor.tokenizer(
                 base_output.input_text,
                 return_tensors="pt",
@@ -689,9 +786,9 @@ class BaseMultimodalProcessor(ABC):
             return [], input_ids, {}
 
         dict_items, raw_images, raw_audios, raw_videos = [], [], [], []
-        for modality, item in all_items:
+        for modality, item in all_loaded_data:
             if isinstance(item, dict):
-                dict_items.append(item)
+                dict_items.append((modality, item))
             elif modality == Modality.IMAGE:
                 raw_images.append(item)
             elif modality == Modality.AUDIO:
@@ -717,12 +814,25 @@ class BaseMultimodalProcessor(ABC):
         else:
             ret = None
 
-        # Handle dict items (already processed)
-        for dict_item in dict_items:
-            all_collected_items.extend(
-                self.collect_mm_items_from_processor_output(dict_item)
-            )
-
+        # Handle dict items (processed or precomputed)
+        for modality, dict_item in dict_items:
+            input_format = dict_item.get("format", None)
+            if input_format == "processor_output":
+                items = self.collect_mm_items_from_processor_output(dict_item)
+                for item in items:
+                    item.format = MultimodalInputFormat.PROCESSOR_OUTPUT
+                all_collected_items.extend(items)
+            elif input_format == "precomputed_embedding":
+                feature = dict_item["feature"]
+                del dict_item["feature"]
+                all_collected_items.append(
+                    MultimodalDataItem(
+                        modality=modality,
+                        feature=feature,
+                        format=MultimodalInputFormat.PRECOMPUTED_EMBEDDING,
+                        model_specific_data=dict_item,
+                    )
+                )
         # Fallback tokenization if no raw items were processed
         if input_ids is None:
             input_ids = self._processor.tokenizer(

python/sglang/srt/multimodal/processors/llava.py

@@ -1,5 +1,5 @@
 import asyncio
-from typing import List, Optional, Union
+from typing import Dict, List, Optional, Union
 
 import numpy as np
 from transformers.models.auto.processing_auto import (
@@ -106,6 +106,32 @@ class LlavaImageProcessor(BaseMultimodalProcessor):
                 self._processor.image_processor,
             )
 
+    def _process_precomputed_image_data(self, image_data: List[Dict]) -> Dict:
+        mm_items = []
+        for item in image_data:
+            # Infer size logic...
+            if "image_sizes" not in item:
+                if "pixel_values" in item:
+                    pv = item["pixel_values"]
+                    # Handle simplified if/else
+                    h, w = (
+                        (pv.shape[2], pv.shape[3])
+                        if len(pv.shape) == 4
+                        else (pv.shape[1], pv.shape[2])
+                    )
+                    item["image_sizes"] = [(w, h)]
+                else:
+                    item["image_sizes"] = [(336, 336)]
+
+            mm_items.append(
+                MultimodalDataItem(
+                    feature=item["feature"],
+                    modality=Modality.IMAGE,
+                    model_specific_data=item,
+                )
+            )
+        return {"mm_items": mm_items}
+
     async def process_mm_data_async(
         self,
         image_data: List[Union[str, bytes, ImageData]],
@@ -114,6 +140,17 @@ class LlavaImageProcessor(BaseMultimodalProcessor):
         *args,
         **kwargs,
     ):
+        # FIX: Handle precomputed embeddings (dictionaries)
+        # If the input is already a dictionary, we skip the CPU image processor.
+        # We also need to infer 'image_sizes' from 'pixel_values' if missing,
+        # because pad_input_ids requires it.
+        if (
+            isinstance(image_data, list)
+            and len(image_data) > 0
+            and isinstance(image_data[0], dict)
+        ):
+            return self._process_precomputed_image_data(image_data)
+
         modalities = request_obj.modalities or ["image"]
         aspect_ratio = getattr(self.hf_config, "image_aspect_ratio", None)
         grid_pinpoints = (
@@ -180,6 +217,8 @@ class LlavaMultimodalProcessor(BaseMultimodalProcessor):
     models = [LlavaForConditionalGeneration, Mistral3ForConditionalGeneration]
 
     def _get_sgl_processor_cls(self, model_type: str):
+        if model_type == "clip_vision_model":
+            return LlavaImageProcessor
         if hf_name := HF_MAPPING_NAMES.get(model_type):
             sgl_mm_processor_set = sgl_mm_processor_utils.PROCESSOR_MAPPING.values()
             sgl_processor_cls = list(

python/sglang/srt/multimodal/processors/qwen_vl.py

@@ -347,14 +347,30 @@ class QwenVLImageProcessor(SGLangBaseProcessor):
                     audio_item.feature_attention_mask, dim=1
                 )
 
-        second_per_grid_ts = getattr(ret, "second_per_grid_ts", None) or getattr(
-            ret, "video_second_per_grid", None
-        )
+        second_per_grid_ts = getattr(ret, "second_per_grid_ts", None)
+        if second_per_grid_ts is None:
+            second_per_grid_ts = getattr(ret, "video_second_per_grid", None)
 
         process_time = time.perf_counter()
 
         input_ids = input_ids.flatten()
 
+        image_grid_thw = None
+        if hasattr(ret, "image_grid_thw"):
+            image_grid_thw = ret.image_grid_thw
+
+        if image_grid_thw is None and image_data and isinstance(image_data[0], dict):
+            image_grid_thw = image_data[0].get("image_grid_thw")
+
+        video_grid_thw = None
+        if hasattr(ret, "video_grid_thw"):
+            video_grid_thw = ret.video_grid_thw
+
+        if video_grid_thw is None and request_obj.video_data:
+            first_video = request_obj.video_data[0]
+            if isinstance(first_video, dict):
+                video_grid_thw = first_video.get("video_grid_thw")
+
         mrope_positions, mrope_position_delta = MRotaryEmbedding.get_rope_index(
             spatial_merge_size=self.hf_config.vision_config.spatial_merge_size,
             image_token_id=self.mm_tokens.image_token_id,
@@ -364,6 +380,7 @@ class QwenVLImageProcessor(SGLangBaseProcessor):
             tokens_per_second=getattr(
                 self.hf_config.vision_config, "tokens_per_second", None
             ),
+            # use the expanded token ids
             input_ids=input_ids.unsqueeze(0),
             image_grid_thw=getattr(ret, "image_grid_thw", None),
             video_grid_thw=getattr(ret, "video_grid_thw", None),

python/sglang/test/run_eval.py

@@ -39,7 +39,7 @@ def run_eval_once(args, base_url: str, eval_obj: Eval) -> dict:
         base_url=base_url,
         temperature=getattr(args, "temperature", 0.0),
         reasoning_effort=getattr(args, "reasoning_effort", None),
-        extra_body=thinking_kwargs,
+        extra_body=thinking_kwargs if thinking_kwargs else None,
     )
 
     # Run eval