chenchenghao/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

feat: support EPD disaggregation (#12263)

Browse Source 
Co-authored-by: liusy58 <liusy58@linux.alibaba.com>
Co-authored-by: ZhengWG <zwg0606@gmail.com>
Co-authored-by: Nicholas <45984215+liusy58@users.noreply.github.com>
Co-authored-by: Shangming Cai <csmthu@gmail.com>
Co-authored-by: Yuhao Yang <47235274+yhyang201@users.noreply.github.com>
This commit is contained in:
Tianyu Guo
2025-12-14 22:30:08 +08:00
committed by GitHub
co-authored by liusy58 ZhengWG Nicholas Shangming Cai Yuhao Yang
parent a9ce1623cd
commit 9acb21ae27
19 changed files with 1910 additions and 68 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/sglang/srt/models/qwen2_5_vl.py
+35 -21
View File
@@ -558,6 +558,31 @@ class Qwen2_5_VLForConditionalGeneration(nn.Module):
self.pp_group = get_pp_group()
self.config = config
self.use_data_parallel = get_global_server_args().mm_enable_dp_encoder
if not self.config.encoder_only:
self.model = Qwen2Model(
config,
quant_config,
prefix=add_prefix("model", prefix),
)
if self.pp_group.is_last_rank:
if self.pp_group.world_size == 1 and self.config.tie_word_embeddings:
self.lm_head = self.model.embed_tokens
else:
self.lm_head = ParallelLMHead(
self.config.vocab_size,
self.config.hidden_size,
quant_config=quant_config,
prefix=add_prefix("lm_head", prefix),
)
else:
# ranks other than the last rank will have a placeholder layer
self.lm_head = PPMissingLayer()
else:
# encoder_only mode: no language model, so no lm_head needed
self.lm_head = None
self.visual = Qwen2_5_VisionTransformer(
config.vision_config,
norm_eps=getattr(config, "rms_norm_eps", 1e-6),
@@ -569,26 +594,6 @@ class Qwen2_5_VLForConditionalGeneration(nn.Module):
max_context_len=self.config.max_position_embeddings,
)
self.model = Qwen2Model(
config,
quant_config,
prefix=add_prefix("model", prefix),
)
if self.pp_group.is_last_rank:
if self.pp_group.world_size == 1 and self.config.tie_word_embeddings:
self.lm_head = self.model.embed_tokens
else:
self.lm_head = ParallelLMHead(
self.config.vocab_size,
self.config.hidden_size,
quant_config=quant_config,
prefix=add_prefix("lm_head", prefix),
)
else:
# ranks other than the last rank will have a placeholder layer
self.lm_head = PPMissingLayer()
self.is_mrope_enabled = "mrope_section" in self.config.rope_scaling
self.logits_processor = LogitsProcessor(config)
@@ -751,6 +756,7 @@ class Qwen2_5_VLForConditionalGeneration(nn.Module):
layer_id = get_layer_id(name)
if (
layer_id is not None
and hasattr(self, "model")
and hasattr(self.model, "start_layer")
and (
layer_id < self.model.start_layer
@@ -762,6 +768,11 @@ class Qwen2_5_VLForConditionalGeneration(nn.Module):
# Skip loading extra bias for GPTQ models.
if name.endswith(".bias") and name not in params_dict:
continue
# Skip loading visual/language model weights
if (
self.config.encoder_only or self.config.language_only
) and name not in params_dict:
continue
param = params_dict[name]
weight_loader = param.weight_loader
weight_loader(param, loaded_weight, shard_id)
@@ -778,7 +789,10 @@ class Qwen2_5_VLForConditionalGeneration(nn.Module):
if name in params_dict.keys():
param = params_dict[name]
else:
raise ValueError(f"Weight {name} not found in params_dict")
if get_global_server_args().encoder_only:
continue
else:
raise ValueError(f"Weight {name} not found in params_dict")
except KeyError:
print(params_dict.keys())
raise