[BugFix] weight load bug when checkpoint expert.gate and exepert.up_proj are not fused (#13113)

Co-authored-by: yuechguo <yuechguo@amd.com>
This commit is contained in:
Morpheus Guo
2025-11-14 00:35:44 -08:00
committed by GitHub
co-authored by yuechguo
parent 49141df94a
commit e7b57b0d04
3 changed files with 30 additions and 41 deletions
+16 -10
View File
@@ -32,6 +32,8 @@ if _is_cuda:
if _is_npu:
import torch_npu
_use_aiter = get_bool_env_var("SGLANG_USE_AITER") and _is_hip
from sglang.srt.distributed import (
split_tensor_along_last_dim,
tensor_model_parallel_all_gather,
@@ -55,10 +57,6 @@ ROTARY_EMBED_CLASSES = {
"normal": apply_rotary_pos_emb,
}
_use_aiter = get_bool_env_var("SGLANG_USE_AITER") and _is_hip
if _use_aiter:
from aiter import flash_attn_varlen_func as aiter_flash_attn_varlen_func
@dataclasses.dataclass
class SingletonCache:
@@ -348,8 +346,16 @@ class VisionAiterAttention(nn.Module):
self,
**kwargs,
):
if not _use_aiter:
if not _is_hip:
raise Exception("aiter_attn is only available for AMD")
try:
from aiter import flash_attn_varlen_func as aiter_flash_attn_varlen_func
except ImportError as e:
raise ImportError(
"aiter is AMD specific kernel library. Please make sure aiter is installed on your AMD device."
) from e
self.flash_attn_varlen_func = aiter_flash_attn_varlen_func
super().__init__()
def forward(
@@ -375,7 +381,7 @@ class VisionAiterAttention(nn.Module):
seq_lens = cu_seqlens[1:] - cu_seqlens[:-1]
max_seqlen = seq_lens.max().item()
return aiter_flash_attn_varlen_func(
return self.flash_attn_varlen_func(
q=q,
k=k,
v=v,
@@ -590,11 +596,11 @@ class VisionAttention(nn.Module):
backend = "fa3"
else:
backend = "triton_attn"
elif _use_aiter:
if get_device_capability() < (9, 4):
backend = "triton_attn"
else:
elif _is_hip:
if get_device_capability() >= (9, 4) and _use_aiter:
backend = "aiter_attn"
else:
backend = "triton_attn"
else:
backend = "sdpa"
if backend == "fa3" and is_blackwell():