@@ -1217,10 +1217,7 @@ class DeepseekV2AttentionMLA(
|
||||
)
|
||||
|
||||
if rope_scaling and rope_scaling.get("apply_yarn_scaling", True):
|
||||
mscale_all_dim = rope_scaling.get("mscale_all_dim", False)
|
||||
scaling_factor = rope_scaling["factor"]
|
||||
mscale = yarn_get_mscale(scaling_factor, float(mscale_all_dim))
|
||||
self.scaling = self.scaling * mscale * mscale
|
||||
self.scaling = compute_mla_mscale_scaling(rope_scaling, self.scaling)
|
||||
else:
|
||||
self.rotary_emb = None
|
||||
self.use_deepseek_yarn_rope = rope_scaling is not None
|
||||
|
||||
Reference in New Issue
Block a user