[VLM] Replace conv3d proj with linear for GLM4V (#20033)

Co-authored-by: luoyuan.luo <luoyuan.luo@antgroup.com>

python/sglang/srt/models/glm4v.py

@@ -211,16 +211,26 @@ class Glm4vVisionPatchEmbed(nn.Module):
             bias=True,
         )
 
-    def forward(self, x: torch.Tensor) -> torch.Tensor:
-        x = x.view(
-            -1,
-            self.in_channels,
-            self.temporal_patch_size,
-            self.patch_size,
-            self.patch_size,
+        k = self.in_channels * self.temporal_patch_size * self.patch_size**2
+        self.linear = nn.Linear(
+            in_features=k,
+            out_features=self.hidden_size,
+            bias=True,
+            dtype=self.proj.weight.dtype,
         )
-        x = self.proj(x).view(-1, self.hidden_size)
-        return x
+
+    def copy_conv3d_weight_to_linear(self):
+        # Call this after weight loading
+        with torch.no_grad():
+            self.linear.weight.copy_(self.proj.weight.view(self.hidden_size, -1))
+            self.linear.bias.copy_(self.proj.bias)
+        del self.proj
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        # After copy_conv3d_weight_to_linear(), self.linear exists and
+        # self.proj has been deleted.  Input x is already 2-D:
+        #   (num_patches, C * T * P * P)
+        return self.linear(x)
 
 
 class Glm4vPatchMerger(nn.Module):
@@ -446,10 +456,16 @@ class Glm4vVisionModel(nn.Module):
 
     @property
     def dtype(self) -> torch.dtype:
+        # After Conv3d to Linear conversion, self.proj is deleted and
+        # self.linear takes its place.
+        if hasattr(self.patch_embed, "linear"):
+            return self.patch_embed.linear.weight.dtype
         return self.patch_embed.proj.weight.dtype
 
     @property
     def device(self) -> torch.device:
+        if hasattr(self.patch_embed, "linear"):
+            return self.patch_embed.linear.weight.device
         return self.patch_embed.proj.weight.device
 
     def rot_pos_emb(
@@ -799,6 +815,7 @@ class Glm4vForConditionalGeneration(nn.Module):
                         self.config, name, loaded_weight
                     )
                 weight_loader(param, loaded_weight)
+        self.visual.patch_embed.copy_conv3d_weight_to_linear()
 
     def get_embed_and_head(self):
         return self.model.embed_tokens.weight, self.lm_head.weight