[VLM] Support Piecewise CUDA Graph for InternVL (#13640)

Co-authored-by: luoyuan.luo <luoyuan.luo@antgroup.com>
This commit is contained in:
Yuan Luo
2025-11-21 13:21:51 +08:00
committed by GitHub
co-authored by luoyuan.luo
parent bfcf15a129
commit 475962a139
5 changed files with 103 additions and 13 deletions
+37 -2
View File
@@ -983,12 +983,47 @@ def should_use_external_mm_preprocess(multimodal_model: nn.Module) -> bool:
cls_name = multimodal_model.__class__.__name__
qwen_vl_classes = {
external_mm_preprocess_classes = {
"Qwen2VLForConditionalGeneration",
"Qwen2_5_VLForConditionalGeneration",
"InternVLChatModel",
}
return cls_name in qwen_vl_classes
return cls_name in external_mm_preprocess_classes
def resolve_external_mm_data_embedding_funcs(
multimodal_model: nn.Module,
) -> Optional[Dict[Modality, Callable[[List[MultimodalDataItem]], torch.Tensor]]]:
"""
Resolve the data_embedding_funcs mapping for external_mm_preprocess_routine
based on the given multimodal model. If this function returns None, the
external_mm_preprocess_routine will use its internal default behavior
(for example, for Qwen2_5_VL).
Resolution order:
1. If the model exposes external_mm_data_embedding_funcs explicitly,
adopt it.
2. TODO: Handle special classes with customized mm_data_embedding_funcs
(e.g. Qwen3_VL).
3. If not mapping, return None.
"""
cls_name = multimodal_model.__class__.__name__
# High priority: model provides an explicit mapping attribute.
# Example in InternVLChatModel.__init__:
# self.external_mm_data_embedding_funcs = {
# Modality.IMAGE: self.get_image_feature,
# }
if hasattr(multimodal_model, "external_mm_data_embedding_funcs"):
funcs = getattr(multimodal_model, "external_mm_data_embedding_funcs")
# Allow an empty dict to mean "no data_embedding_funcs are needed".
return funcs or None
# If no mapping is found, return None so that external_mm_preprocess_routine
# can fall back to its default logic.
return None
def external_mm_preprocess_routine(