refactor: minor refactors regarding multimodal processing (#6187)

2025-05-17 22:53:20 -07:00
parent b3f3d610fd
commit 01dd39bac1
15 changed files with 140 additions and 98 deletions
@@ -1520,12 +1520,15 @@ class MiniCPMO(MiniCPMBaseModel):
        slice_start_id: int = mm_input.slice_start_id
        slice_end_id: int = mm_input.slice_end_id

-        media_token_pairs = [
+        data_token_pairs = [
            (im_start_id, im_end_id),
            (slice_start_id, slice_end_id),
            (mm_input.audio_start_id, mm_input.audio_end_id),
        ]
-        pattern = MultiModalityDataPaddingPatternTokenPairs(media_token_pairs)
+        data_start_token_ids = [im_start_id, mm_input.audio_start_id]
+        pattern = MultiModalityDataPaddingPatternTokenPairs(
+            data_token_pairs=data_token_pairs, data_start_token_ids=data_start_token_ids
+        )

        return pattern.pad_input_tokens(input_ids, mm_input)