[NemotronH] Add latent MoE support (#16227)

Signed-off-by: Roi Koren <roik@nvidia.com>
2026-01-02 22:08:58 +08:00
parent 5062537b67
commit b021332339
23 changed files with 2957 additions and 2 deletions
@@ -194,6 +194,7 @@ class NemotronHConfig(PretrainedConfig):
        n_shared_experts=1,
        moe_intermediate_size=7688,
        moe_shared_expert_intermediate_size=7688,
+        moe_latent_size=None,
        num_experts_per_tok=2,
        routed_scaling_factor=1.0,
        n_group=1,
@@ -259,6 +260,7 @@ class NemotronHConfig(PretrainedConfig):
        self.n_shared_experts = n_shared_experts
        self.moe_intermediate_size = moe_intermediate_size
        self.moe_shared_expert_intermediate_size = moe_shared_expert_intermediate_size
+        self.moe_latent_size = moe_latent_size
        self.num_experts_per_tok = num_experts_per_tok
        self.routed_scaling_factor = routed_scaling_factor
        self.n_group = n_group