[Refactor] Multimodal data processing for VLM (#6659)
Signed-off-by: Xinyuan Tong <justinning0323@outlook.com>
This commit is contained in:
@@ -188,7 +188,7 @@ class MultimodalDataItem:
|
||||
# the real data, pixel_values or audio_features
|
||||
# data: Union[List[torch.Tensor], List[np.ndarray]]
|
||||
pixel_values: Union[torch.Tensor, np.ndarray] = None
|
||||
image_grid_thws: Union[torch.Tensor, np.ndarray] = None
|
||||
image_grid_thw: Union[torch.Tensor, np.ndarray] = None
|
||||
video_grid_thws: Union[torch.Tensor, np.ndarray] = None
|
||||
|
||||
image_emb_mask: Optional[torch.Tensor] = None
|
||||
@@ -198,6 +198,9 @@ class MultimodalDataItem:
|
||||
# [num_images, (n, w, h)]
|
||||
tgt_size: Tuple[int, int] = None
|
||||
|
||||
# kimi-vl related
|
||||
image_grid_hws: Optional[List[torch.Tensor]] = None
|
||||
|
||||
audio_features: Union[torch.Tensor, np.ndarray] = None
|
||||
audio_feature_lens: Optional[List[torch.Tensor]] = None
|
||||
audio_offsets: Optional[List[Tuple[int, int]]] = None
|
||||
|
||||
Reference in New Issue
Block a user