[3/n] chore: decouple AWQ implementation from vLLM dependency (#8113)
Co-authored-by: AniZpZ <zhuangsen.zp@antgroup.com>
This commit is contained in:
@@ -355,6 +355,7 @@ class DeepseekV2MoE(nn.Module):
|
||||
self.shared_experts.gate_up_proj.quant_method, "quant_config"
|
||||
) and self.shared_experts.gate_up_proj.quant_method.quant_config.get_name() in {
|
||||
"awq",
|
||||
"awq_marlin",
|
||||
"moe_wna16",
|
||||
}
|
||||
self.shared_experts_is_int8 = (
|
||||
@@ -929,7 +930,7 @@ class DeepseekV2AttentionMLA(nn.Module):
|
||||
has_fused_proj
|
||||
and hasattr(self.fused_qkv_a_proj_with_mqa.quant_method, "quant_config")
|
||||
and self.fused_qkv_a_proj_with_mqa.quant_method.quant_config.get_name()
|
||||
in {"awq", "moe_wna16"}
|
||||
in {"awq", "awq_marlin", "moe_wna16"}
|
||||
)
|
||||
self.use_min_latency_fused_a_gemm = (
|
||||
has_fused_proj
|
||||
@@ -2551,6 +2552,7 @@ class DeepseekV2ForCausalLM(nn.Module):
|
||||
cat_dim = 0
|
||||
if self.quant_config is not None and (
|
||||
self.quant_config.get_name() == "awq"
|
||||
or self.quant_config.get_name() == "awq_marlin"
|
||||
or self.quant_config.get_name() == "moe_wna16"
|
||||
):
|
||||
cat_dim = 1
|
||||
|
||||
Reference in New Issue
Block a user