fix: Fix KTransformers hybrid inference with int8 quantization and format (#12536)

2025-11-03 04:59:39 -08:00
parent ab8b83f71d
commit 14d8064803
2 changed files with 3 additions and 4 deletions
@@ -751,12 +751,10 @@ class DeepseekV2MoE(nn.Module):
            # router_logits: (num_tokens, n_experts)
            router_logits = self.gate(hidden_states, gemm_output_zero_allocator)
            topk_output = self.topk(hidden_states, router_logits)
-            if isinstance(
+            final_hidden_states = self.experts(hidden_states, topk_output)
+            if not _is_cuda or isinstance(
                self.experts.quant_method, CompressedTensorsWNA16AMXEPMoEMethod
            ):
-                topk_output.topk_weights.mul_(self.routed_scaling_factor)
-            final_hidden_states = self.experts(hidden_states, topk_output)
-            if not _is_cuda:
                final_hidden_states *= self.routed_scaling_factor

        current_stream.wait_stream(self.alt_stream)