[NPU] support model skywork-reward-gemma2-2-27B-v0.2 (#16947)
Co-authored-by: cy <chenyang08056032@163.com>
This commit is contained in:
@@ -225,6 +225,11 @@ class AscendAttnBackend(AttentionBackend):
|
||||
self.q_head_dim = self.qk_rope_head_dim + self.qk_nope_head_dim
|
||||
else:
|
||||
self.use_alibi = getattr(model_runner.model_config, "use_alibi", False)
|
||||
if (
|
||||
"Gemma2ForSequenceClassification"
|
||||
in model_runner.model_config.hf_config.architectures
|
||||
):
|
||||
self.use_native_sdpa = True
|
||||
self.native_attn = AscendTorchNativeAttnBackend()
|
||||
self.graph_metadata = {}
|
||||
self.max_context_len = model_runner.model_config.context_len
|
||||
@@ -821,10 +826,12 @@ class AscendAttnBackend(AttentionBackend):
|
||||
|
||||
# there are some accuracy issues in cross attention scene to use torch_npu._npu_flash_attention_qlens
|
||||
# forward_batch.encoder_lens is not None in cross attention scend, we add native attn to solve accuracy issues
|
||||
# Model skywork-reward-gemma2-2-27B also suffers from precision anomalies, thus the torch native backend becomes beneficial approach.
|
||||
if (
|
||||
layer.qk_head_dim <= 128
|
||||
and causal
|
||||
and forward_batch.encoder_lens is None
|
||||
and not getattr(self, "use_native_sdpa", False)
|
||||
):
|
||||
if not self.use_alibi:
|
||||
query = q.reshape(-1, layer.tp_q_head_num * layer.qk_head_dim)
|
||||
|
||||
Reference in New Issue
Block a user