chenchenghao/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 11 Packages Projects Releases Wiki Activity

[Glm46v] Bug fix for accuracy drop and unable to launch server (#14585)

Browse Source 
Co-authored-by: yhyang201 <yhyang201@gmail.com>
Co-authored-by: zRzRzRzRzRzRzR <2448370773@qq.com>
Co-authored-by: Minglei Zhu <mingleizhu1122@gmail.com>
This commit is contained in:
Binyao Jiang
2025-12-07 23:45:02 -08:00
committed by GitHub
parent a2ca9bd4f1
commit cf0478d602
12 changed files with 308 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
python/pyproject.toml
View File

@@ -65,7 +65,7 @@ dependencies = [
"torch_memory_saver==0.0.9",
"torch==2.9.1",
"torchaudio==2.9.1",
"torchcodec==0.7.0 ; sys_platform != 'linux' or (sys_platform == 'linux' and platform_machine != 'aarch64' and platform_machine != 'arm64' and platform_machine != 'armv7l')", # torchcodec does not exist in those systems. If not provided, transformer will use torchvision instead by default.
"torchcodec==0.8.0 ; sys_platform != 'linux' or (sys_platform == 'linux' and platform_machine != 'aarch64' and platform_machine != 'arm64' and platform_machine != 'armv7l')", # torchcodec does not exist in those systems. If not provided, transformer will use torchvision instead by default.
"torchvision",
"torchao==0.9.0",
"tqdm",

4
python/sglang/srt/configs/qwen3_omni.py
View File

@@ -1,6 +1,5 @@
from transformers import PretrainedConfig
from transformers.configuration_utils import layer_type_validation
from transformers.modeling_rope_utils import rope_config_validation
from sglang.utils import logger
@@ -168,7 +167,6 @@ class Qwen3OmniMoeTextConfig(PretrainedConfig):
# BC: if there is a 'type' field, move it to 'rope_type'.
if self.rope_scaling is not None and "type" in self.rope_scaling:
self.rope_scaling["rope_type"] = self.rope_scaling["type"]
rope_config_validation(self)
# MoE arguments
self.decoder_sparse_step = decoder_sparse_step
@@ -311,7 +309,6 @@ class Qwen3OmniMoeTalkerCodePredictorConfig(PretrainedConfig):
# BC: if there is a 'type' field, move it to 'rope_type'.
if self.rope_scaling is not None and "type" in self.rope_scaling:
self.rope_scaling["rope_type"] = self.rope_scaling["type"]
rope_config_validation(self)
self.layer_types = layer_types
if self.layer_types is None:
@@ -405,7 +402,6 @@ class Qwen3OmniMoeTalkerTextConfig(PretrainedConfig):
# BC: if there is a 'type' field, move it to 'rope_type'.
if self.rope_scaling is not None and "type" in self.rope_scaling:
self.rope_scaling["rope_type"] = self.rope_scaling["type"]
rope_config_validation(self)
# MoE arguments
self.decoder_sparse_step = decoder_sparse_step

5
python/sglang/srt/configs/qwen3_vl.py
View File

@@ -1,5 +1,4 @@
from transformers import PretrainedConfig
from transformers.modeling_rope_utils import rope_config_validation
class Qwen3VLVisionConfig(PretrainedConfig):
@@ -187,8 +186,6 @@ class Qwen3VLTextConfig(PretrainedConfig):
self.attention_bias = attention_bias
self.attention_dropout = attention_dropout
rope_config_validation(self, ignore_keys={"mrope_section", "mrope_interleaved"})
super().__init__(tie_word_embeddings=tie_word_embeddings, **kwargs)
@@ -450,8 +447,6 @@ class Qwen3VLMoeTextConfig(PretrainedConfig):
self.rope_scaling = rope_scaling
self.head_dim = head_dim or hidden_size // num_attention_heads
rope_config_validation(self, ignore_keys={"mrope_section", "mrope_interleaved"})
# MoE arguments
self.decoder_sparse_step = decoder_sparse_step
self.moe_intermediate_size = moe_intermediate_size

1
python/sglang/srt/models/glm4_moe.py
View File

@@ -361,6 +361,7 @@ class Glm4MoeSparseMoeBlock(nn.Module):
if get_global_server_args().disable_shared_experts_fusion
else config.n_shared_experts
)
self.config = config
self.layer_id = layer_id
self.alt_stream = alt_stream

22
python/sglang/srt/models/glm4v.py
View File

@@ -123,6 +123,7 @@ class Glm4vVisionBlock(nn.Module):
num_heads: int,
quant_config: Optional[QuantizationConfig] = None,
prefix: str = "",
attn_qkv_bias: bool = True,
num_dummy_heads: int = 0,
rms_norm_eps: float = 1e-5,
use_data_parallel: bool = False,
@@ -136,7 +137,8 @@ class Glm4vVisionBlock(nn.Module):
num_heads=num_heads,
projection_size=dim,
use_qkv_parallel=True,
proj_bias=True,
proj_bias=False,
qkv_bias=attn_qkv_bias,
flatten_batch=True,
quant_config=quant_config,
prefix=add_prefix("attn", prefix),
@@ -440,6 +442,7 @@ class Glm4vVisionModel(nn.Module):
quant_config=quant_config,
prefix=add_prefix(f"blocks.{layer_idx}", prefix),
rms_norm_eps=vision_config.rms_norm_eps,
attn_qkv_bias=vision_config.attention_bias,
use_data_parallel=use_data_parallel,
)
for layer_idx in range(depth)
@@ -623,14 +626,27 @@ class Glm4vForConditionalGeneration(nn.Module):
self.visual.dtype
)
video_grid_thw = torch.concat([item.video_grid_thw for item in items], dim=0)
# reshape video_grid_thw -> [b, 3] -> [1, h, w] * frames
temp_frames_hw = []
for t, h, w in video_grid_thw:
repeated_row = (
torch.tensor([1, h.item(), w.item()]).unsqueeze(0).repeat(t, 1)
)
temp_frames_hw.append(repeated_row)
flattened_video_grid_thw = torch.cat(temp_frames_hw, dim=0)
assert pixel_values.dim() == 2, pixel_values.dim()
assert video_grid_thw.dim() == 2, video_grid_thw.dim()
if self.use_data_parallel:
return run_dp_sharded_mrope_vision_model(
self.visual, pixel_values, video_grid_thw.tolist(), rope_type="rope_3d"
self.visual,
pixel_values,
flattened_video_grid_thw.tolist(),
rope_type="rope_3d",
)
else:
video_embeds = self.visual(pixel_values, grid_thw=video_grid_thw)
video_embeds = self.visual(pixel_values, grid_thw=flattened_video_grid_thw)
return video_embeds
def get_input_embeddings(self):

83
python/sglang/srt/models/glm4v_moe.py
View File

@@ -6,21 +6,28 @@ import torch
import torch.nn as nn
from transformers.models.glm4v_moe.configuration_glm4v_moe import Glm4vMoeConfig
from sglang.srt.distributed import get_tensor_model_parallel_world_size
from sglang.srt.distributed import (
get_moe_expert_parallel_world_size,
get_tensor_model_parallel_world_size,
)
from sglang.srt.distributed.parallel_state import get_pp_group
from sglang.srt.layers.attention import vision_utils
from sglang.srt.layers.logits_processor import LogitsProcessor
from sglang.srt.layers.moe import get_moe_a2a_backend
from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE
from sglang.srt.layers.pooler import Pooler, PoolingType
from sglang.srt.layers.quantization.base_config import QuantizationConfig
from sglang.srt.layers.utils import PPMissingLayer
from sglang.srt.layers.vocab_parallel_embedding import ParallelLMHead
from sglang.srt.model_loader.weight_utils import default_weight_loader
from sglang.srt.models.glm4_moe import Glm4MoeModel
from sglang.srt.models.glm4v import Glm4vForConditionalGeneration, Glm4vVisionModel
from sglang.srt.server_args import get_global_server_args
from sglang.srt.utils import add_prefix, is_cuda
from sglang.srt.utils import add_prefix, get_device_sm, is_cuda, log_info_on_rank0
from sglang.srt.utils.hf_transformers_utils import get_processor
_is_cuda = is_cuda()
_device_sm = get_device_sm()
logger = logging.getLogger(__name__)
@@ -36,15 +43,14 @@ class Glm4vMoeForConditionalGeneration(Glm4vForConditionalGeneration):
) -> None:
nn.Module.__init__(self)
self.pp_group = get_pp_group()
self.config = config
self.use_data_parallel = get_global_server_args().mm_enable_dp_encoder
vision_utils.update_vit_attn_dummy_heads_config(self.config)
self.tp_size = get_tensor_model_parallel_world_size()
self.quant_config = quant_config
self.num_fused_shared_experts = (
0
if get_global_server_args().disable_shared_experts_fusion
else config.n_shared_experts
)
self.num_fused_shared_experts = 0
self.determine_num_fused_shared_experts()
self.model = Glm4MoeModel(
config,
@@ -55,15 +61,24 @@ class Glm4vMoeForConditionalGeneration(Glm4vForConditionalGeneration):
config.vision_config,
quant_config=quant_config,
prefix=add_prefix("visual", prefix),
use_data_parallel=self.use_data_parallel,
)
self.lm_head = ParallelLMHead(
config.vocab_size,
config.hidden_size,
quant_config=quant_config,
prefix=add_prefix("lm_head", prefix),
use_attn_tp_group=get_global_server_args().enable_dp_lm_head,
)
if self.pp_group.is_last_rank:
if self.pp_group.world_size == 1 and self.config.tie_word_embeddings:
self.lm_head = self.model.embed_tokens
else:
self.lm_head = ParallelLMHead(
config.vocab_size,
config.hidden_size,
quant_config=quant_config,
prefix=add_prefix("lm_head", prefix),
use_attn_tp_group=get_global_server_args().enable_dp_lm_head,
)
else:
# ranks other than the last rank will have a placeholder layer
self.lm_head = PPMissingLayer()
self.logits_processor = LogitsProcessor(config)
self.pooler = Pooler(pooling_type=PoolingType.LAST, normalize=True)
self.is_mrope_enabled = "mrope_section" in self.config.rope_scaling
@@ -71,6 +86,36 @@ class Glm4vMoeForConditionalGeneration(Glm4vForConditionalGeneration):
# For EAGLE3 support
self.capture_aux_hidden_states = False
def determine_num_fused_shared_experts(self):
if get_global_server_args().disable_shared_experts_fusion:
return
disable_reason = None
if not getattr(self.config, "n_shared_experts", None):
disable_reason = "No shared experts are defined in the config."
elif not _is_cuda:
disable_reason = "Shared experts fusion currently requires CUDA devices."
elif _is_cuda and (_device_sm is not None) and (_device_sm < 80):
disable_reason = "Shared experts fusion requires SM80 or newer GPUs."
elif get_moe_expert_parallel_world_size() > 1:
disable_reason = "Shared experts fusion is not supported together with expert parallelism yet."
elif get_moe_a2a_backend().is_deepep():
disable_reason = "Shared experts fusion is not supported when Deepep MoE backend is enabled."
if disable_reason is not None:
get_global_server_args().disable_shared_experts_fusion = True
log_info_on_rank0(
logger,
f"{disable_reason} Shared experts fusion optimization is disabled.",
)
return
self.num_fused_shared_experts = self.config.n_shared_experts
assert (
self.num_fused_shared_experts == 1
), "Only 1 fused shared expert is supported for Glm4vMoeForConditionalGeneration"
log_info_on_rank0(logger, "Shared experts fusion optimization enabled.")
def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]], is_nextn=False):
if is_nextn:
if hasattr(self.config, "num_nextn_predict_layers"):
@@ -98,7 +143,7 @@ class Glm4vMoeForConditionalGeneration(Glm4vForConditionalGeneration):
ckpt_gate_proj_name="gate_proj",
ckpt_down_proj_name="down_proj",
ckpt_up_proj_name="up_proj",
num_experts=self.config.n_routed_experts,
num_experts=self.config.n_routed_experts + self.num_fused_shared_experts,
)
if is_nextn:
@@ -115,6 +160,13 @@ class Glm4vMoeForConditionalGeneration(Glm4vForConditionalGeneration):
for name, loaded_weight in weights:
weight_names.append(name)
if self.num_fused_shared_experts > 0 and "mlp.shared_experts" in name:
# Shared expert becomes expert ID = n_routed_experts
name = name.replace(
"mlp.shared_experts",
f"mlp.experts.{self.config.n_routed_experts}",
)
if not is_nextn:
if hasattr(self.config, "num_nextn_predict_layers"):
num_nextn_layers = self.config.num_nextn_predict_layers
@@ -150,6 +202,7 @@ class Glm4vMoeForConditionalGeneration(Glm4vForConditionalGeneration):
name = name.replace("model.visual.", "visual.")
if "rotary_emb.inv_freq" in name:
continue
for param_name, weight_name, shard_id in stacked_params_mapping:
# Skip non-stacked layers and experts (experts handled below).
if weight_name not in name:

8
python/sglang/srt/sampling/custom_logit_processor.py
View File

@@ -112,6 +112,14 @@ class ThinkingBudgetLogitProcessor(CustomLogitProcessor):
return logits
class Glm4MoeThinkingBudgetLogitProcessor(ThinkingBudgetLogitProcessor):
"""A logit processor that controls the length of thinking for GLM-4.5 / GLM-4.6 / GLM-4.5V / GLM-4.6V models."""
THINKING_START_TOKEN_ID: int = 151350
THINKING_END_TOKEN_ID: int = 151351
NEW_LINE_TOKEN_ID: int = 198
class Qwen3ThinkingBudgetLogitProcessor(ThinkingBudgetLogitProcessor):
"""A logit processor that controls the length of thinking for Qwen3 models."""

1
python/sglang/srt/utils/common.py
View File

@@ -2716,6 +2716,7 @@ def is_fa3_default_architecture(hf_config):
"Qwen3ForCausalLM",
"Qwen3MoeForCausalLM",
"Glm4MoeForCausalLM",
"Glm4vForConditionalGeneration",
"Glm4vMoeForConditionalGeneration",
"Step3VLForConditionalGeneration",
}