[Bugifx] qwen3 rope parameter compatibility (#20931)
Co-authored-by: Xinyuan Tong <xinyuantong.cs@gmail.com>
This commit is contained in:
@@ -78,6 +78,7 @@ from sglang.srt.utils import (
|
||||
is_non_idle_and_non_empty,
|
||||
is_npu,
|
||||
)
|
||||
from sglang.srt.utils.hf_transformers_utils import get_rope_config
|
||||
|
||||
_is_cuda = is_cuda()
|
||||
|
||||
@@ -566,7 +567,7 @@ class Qwen3MoeAttention(nn.Module):
|
||||
def apply_qk_norm_rope(self, qkv, positions, forward_batch):
|
||||
use_fused = self.use_fused_qk_norm_rope and qkv.dtype == torch.bfloat16
|
||||
if use_fused:
|
||||
theta = self.config.rope_parameters["rope_theta"]
|
||||
theta = self.rope_theta
|
||||
positions = (
|
||||
positions.view(-1).to(dtype=torch.int32, device=qkv.device).contiguous()
|
||||
)
|
||||
@@ -691,8 +692,8 @@ class Qwen3MoeDecoderLayer(nn.Module):
|
||||
super().__init__()
|
||||
self.config = config
|
||||
self.hidden_size = config.hidden_size
|
||||
rope_theta = config.rope_parameters["rope_theta"]
|
||||
rope_scaling = config.rope_parameters
|
||||
rope_theta, rope_scaling = get_rope_config(config)
|
||||
self.rope_theta = rope_theta
|
||||
max_position_embeddings = getattr(config, "max_position_embeddings", 8192)
|
||||
head_dim = getattr(
|
||||
config, "head_dim", config.hidden_size // config.num_attention_heads
|
||||
|
||||
Reference in New Issue
Block a user