update pre-commit config (#18860)
This commit is contained in:
@@ -222,7 +222,7 @@ class TestLayerNorm(CustomTestCase):
|
||||
x = x + residual.to(torch.float32)
|
||||
residual = x.to(orig_dtype)
|
||||
|
||||
(variance, mean) = torch.var_mean(x, dim=-1, keepdim=True, correction=0)
|
||||
variance, mean = torch.var_mean(x, dim=-1, keepdim=True, correction=0)
|
||||
x = (x - mean) * torch.rsqrt(variance + variance_epsilon)
|
||||
x = x.to(orig_dtype) * weight
|
||||
if residual is None:
|
||||
|
||||
@@ -39,8 +39,8 @@ def fix_query_key_value_ordering_reshape_cat(
|
||||
]
|
||||
# [b, sq, ng, (hn + hn + np/ng * hn + np/ng + np/ng)]
|
||||
# --> [b, sq, ng, hn], [b, sq, ng, hn], [b, sq, ng, np/ng * hn], [b, sq, ng, np/ng * hn], [b, sq, ng, np/ng], [b, sq, ng, np/ng]
|
||||
(query, key, value, z) = torch.split(mixed_qkvz, split_arg_list_qkvz, dim=2)
|
||||
(b, a) = torch.split(mixed_ba, split_arg_list_ba, dim=2)
|
||||
query, key, value, z = torch.split(mixed_qkvz, split_arg_list_qkvz, dim=2)
|
||||
b, a = torch.split(mixed_ba, split_arg_list_ba, dim=2)
|
||||
|
||||
# [b, sq, ng, np/ng * hn] -> [b, sq, np, hn]
|
||||
value = value.reshape(value.size(0), -1, head_v_dim)
|
||||
|
||||
Reference in New Issue
Block a user