chenchenghao/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 11 Packages Projects Releases Wiki Activity

support llama4 eagle3 (#6985)

Browse Source 
Co-authored-by: shuaills <shishuaiuoe@gmail.com>
Co-authored-by: Shenggui Li <somerlee.9@gmail.com>
Co-authored-by: Yingyi Huang <yingyihuang2000@outlook.com>
Co-authored-by: yizhang2077 <1109276519@qq.com>
This commit is contained in:
lukec
2025-07-01 13:34:10 +08:00
committed by GitHub
parent 637bfee448
commit 886d344964
7 changed files with 138 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
python/sglang/srt/model_executor/model_runner.py
View File

@@ -306,7 +306,26 @@ class ModelRunner:
# auxiliary hidden capture mode. TODO: expose this to server args?
if self.spec_algorithm.is_eagle3() and not self.is_draft_worker:
self.model.set_eagle3_layers_to_capture()
# load draft config
draft_model_config = ModelConfig.from_server_args(
server_args,
model_path=(server_args.speculative_draft_model_path),
is_draft_model=True,
)
try:
# get the aux layer from draft model config
eagle_config = getattr(
draft_model_config.hf_config, "eagle_config", None
)
eagle_aux_hidden_state_layer_ids = eagle_config[
"eagle_aux_hidden_state_layer_ids"
]
except:
# if there is no aux layer, set to None
eagle_aux_hidden_state_layer_ids = None
self.model.set_eagle3_layers_to_capture(eagle_aux_hidden_state_layer_ids)
def model_specific_adjustment(self):
server_args = self.server_args