[GLM4.1V and GLM4.5V] Add vision transformer num_dummy_head support: max tp=4 -> max tp=8 (#9059)

2025-08-18 14:40:13 -07:00
parent 98b44e9e56
commit c2fbf60f39
9 changed files with 150 additions and 102 deletions
@@ -117,6 +117,7 @@ class Qwen2_5_VisionBlock(nn.Module):
        attn_implementation: Optional[str] = None,
        quant_config: Optional[QuantizationConfig] = None,
        prefix: str = "",
+        num_dummy_heads: int = 0,
    ) -> None:
        super().__init__()
        if norm_layer is None:
@@ -157,6 +158,7 @@ class Qwen2_5_VisionBlock(nn.Module):
            flatten_batch=flatten_batch,
            quant_config=quant_config,
            prefix=add_prefix("attn", prefix),
+            num_dummy_heads=num_dummy_heads,
        )
        self.mlp = Qwen2_5_VLMLP(
            dim,