diff --git a/python/sglang/srt/layers/layernorm.py b/python/sglang/srt/layers/layernorm.py index b07164c53..7bef9d2ab 100644 --- a/python/sglang/srt/layers/layernorm.py +++ b/python/sglang/srt/layers/layernorm.py @@ -125,9 +125,8 @@ class RMSNorm(MultiPlatformOp): # (hidden_states+residual)+post_residual_addition != hidden_states+(residual+post_residual_addition), # we probably need to add another parameter to fused_add_rmsnorm post_residual_addition = kwargs.get("post_residual_addition") - residual = residual + ( - post_residual_addition if post_residual_addition is not None else 0.0 - ) + if post_residual_addition is not None: + residual = residual + post_residual_addition fused_add_rmsnorm(x, residual, self.weight.data, self.variance_epsilon) return x, residual out = rmsnorm(x, self.weight.data, self.variance_epsilon)