Support qwen3 deepep (#6120)

2025-05-22 11:04:45 -07:00
parent d71f3f0a2a
commit fc0e3b9174
2 changed files with 125 additions and 8 deletions
@@ -607,7 +607,10 @@ class Qwen2MoeModel(nn.Module):
            )
        else:
            if hidden_states.shape[0] != 0:
-                hidden_states, _ = self.norm(hidden_states, residual)
+                if residual is None:
+                    hidden_states = self.norm(hidden_states)
+                else:
+                    hidden_states, _ = self.norm(hidden_states, residual)
        return hidden_states