chenchenghao/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 11 Packages Projects Releases Wiki Activity

Deepseekv32 compatibility with transformers v5 (#18297)

Browse Source 
Signed-off-by: Xinyuan Tong <xinyuantong.cs@gmail.com>
Co-authored-by: Xiaoyu Zhang <35585791+BBuf@users.noreply.github.com>
Co-authored-by: Baizhou Zhang <sobereddiezhang@gmail.com>
This commit is contained in:
Xinyuan Tong
2026-02-10 14:50:40 +08:00
committed by GitHub
co-authored by Xiaoyu Zhang Baizhou Zhang
parent 0b15f19927
commit e8a2c13380
5 changed files with 33 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/sglang/srt/models/deepseek_v2.py
+13 -4
View File
@@ -1150,6 +1150,7 @@ class DeepseekV2AttentionMLA(nn.Module, DeepseekMHAForwardMixin):
)
if self.use_nsa:
is_neox_style = not getattr(config, "indexer_rope_interleave", False)
self.indexer = Indexer(
hidden_size=hidden_size,
index_n_heads=get_nsa_index_n_heads(config),
@@ -1162,6 +1163,7 @@ class DeepseekV2AttentionMLA(nn.Module, DeepseekMHAForwardMixin):
scale_fmt="ue8m0",
block_size=128,
rope_scaling=rope_scaling,
is_neox_style=is_neox_style,
prefix=add_prefix("indexer", prefix),
quant_config=quant_config,
layer_id=layer_id,
@@ -1191,13 +1193,14 @@ class DeepseekV2AttentionMLA(nn.Module, DeepseekMHAForwardMixin):
self.kv_a_layernorm = RMSNorm(self.kv_lora_rank, eps=config.rms_norm_eps)
if not skip_rope:
is_neox_style = not getattr(config, "rope_interleave", True)
self.rotary_emb = get_rope_wrapper(
qk_rope_head_dim,
rotary_dim=qk_rope_head_dim,
max_position=max_position_embeddings,
base=rope_theta,
rope_scaling=rope_scaling,
is_neox_style=False,
is_neox_style=is_neox_style,
device=get_global_server_args().device,
)
@@ -2227,9 +2230,15 @@ class DeepseekV2DecoderLayer(nn.Module):
super().__init__()
self.hidden_size = config.hidden_size
self.config = config
rope_theta = getattr(config, "rope_theta", 10000)
rope_scaling = getattr(config, "rope_scaling", None)
max_position_embeddings = getattr(config, "max_position_embeddings", 8192)
if hasattr(config, "rope_parameters"):
rope_theta = config.rope_parameters.get("rope_theta")
assert rope_theta is not None, f"rope_theta not found in config: {config}"
rope_type = config.rope_parameters.get("rope_type")
rope_scaling = config.rope_parameters if rope_type != "default" else None
else:
rope_theta = config.rope_theta
rope_scaling = config.rope_scaling
max_position_embeddings = config.max_position_embeddings
self.speculative_algorithm = SpeculativeAlgorithm.from_string(
get_global_server_args().speculative_algorithm
)