Deepseekv32 compatibility with transformers v5 (#18297)
Signed-off-by: Xinyuan Tong <xinyuantong.cs@gmail.com> Co-authored-by: Xiaoyu Zhang <35585791+BBuf@users.noreply.github.com> Co-authored-by: Baizhou Zhang <sobereddiezhang@gmail.com>
This commit is contained in:
co-authored by
Xiaoyu Zhang
Baizhou Zhang
parent
0b15f19927
commit
e8a2c13380
@@ -300,6 +300,8 @@ class NativeSparseAttnBackend(
|
||||
self.req_to_token = model_runner.req_to_token_pool.req_to_token
|
||||
|
||||
self.use_mha: bool = False
|
||||
# Force NSA prefill to use MLA (i.e. disable MHA_ONE_SHOT), controlled by env var.
|
||||
self._force_attn_forward_mla: bool = envs.SGLANG_NSA_FORCE_MLA.get()
|
||||
self.nsa_prefill_impl: _NSA_IMPL_T = (
|
||||
model_runner.server_args.nsa_prefill_backend
|
||||
)
|
||||
@@ -1837,6 +1839,8 @@ class NativeSparseAttnBackend(
|
||||
)
|
||||
else:
|
||||
self.use_mha = False # Decode/verify always use MLA
|
||||
if self._force_attn_forward_mla:
|
||||
self.use_mha = False
|
||||
|
||||
# Set MLA implementation only if not using MHA
|
||||
if not self.use_mha and self.enable_auto_select_prefill_impl:
|
||||
|
||||
Reference in New Issue
Block a user