[1/N] MoE Refactor: refactor select_experts (#7966)
This commit is contained in:
@@ -29,15 +29,18 @@ class CustomOp(nn.Module):
|
||||
|
||||
self._original_forward_method = self._forward_method
|
||||
# NOTE: Temporarily workaround MoE
|
||||
# The performance of torch.compile on this layer is not always good when bs > 1,
|
||||
# so we decide to only use torch.compile when bs=1
|
||||
if "FusedMoE" in self.__class__.__name__:
|
||||
if num_tokens == 1:
|
||||
from sglang.srt.layers.moe.fused_moe_native import (
|
||||
fused_moe_forward_native,
|
||||
)
|
||||
|
||||
# The performance of torch.compile on this layer is not always good when bs > 1,
|
||||
# so we decide to only use torch.compile when bs =1
|
||||
self._forward_method = fused_moe_forward_native
|
||||
elif "TopK" in self.__class__.__name__:
|
||||
if num_tokens == 1:
|
||||
self._forward_method = self.forward_native
|
||||
else:
|
||||
self._forward_method = self.forward_native
|
||||
self.is_torch_compile = True
|
||||
|
||||
Reference in New Issue
Block a user