vlm: refactor engine vlm params and support processor output as input (#14091)

Co-authored-by: Mick <mickjagger19@icloud.com>
Co-authored-by: zhaochenyang20 <zhaochenyang20@gmail.com>
Co-authored-by: Xinyuan Tong <115166877+JustinTong0323@users.noreply.github.com>
Co-authored-by: BenYao21 <cyao22@asu.edu>
Co-authored-by: minleminzui <minleminzui@gmail.com>
Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Co-authored-by: 赵晨阳 <zhaochen20@outlook.com>

python/sglang/srt/managers/schedule_batch.py

@@ -189,6 +189,12 @@ class Modality(Enum):
         return [Modality.IMAGE, Modality.VIDEO, Modality.AUDIO]
 
 
+class MultimodalInputFormat(Enum):
+    NORMAL = auto()
+    PROCESSOR_OUTPUT = auto()
+    PRECOMPUTED_EMBEDDING = auto()
+
+
 @dataclasses.dataclass
 class MultimodalDataItem:
     """
@@ -204,6 +210,8 @@ class MultimodalDataItem:
     pad_value: int = None
     offsets: Optional[list] = None
 
+    format: MultimodalInputFormat = MultimodalInputFormat.NORMAL
+
     # the raw features returned by processor, e.g. pixel_values or audio_features
     feature: Union[torch.Tensor, np.ndarray] = None
     # the precomputed embeddings, passed as final encoder embeddings
@@ -276,6 +284,9 @@ class MultimodalDataItem:
         ...
         # TODO
 
+    def is_precomputed_embedding(self):
+        return self.format == MultimodalInputFormat.PRECOMPUTED_EMBEDDING
+
     @staticmethod
     def from_dict(obj: dict):
         kwargs = dict(obj)