Reuse draft MTP indexer topk on spec v1 EAGLE
Spec v1 EAGLE now follows the same model-config-gated MTP index reuse contract as spec v2. This lets models that declare index_share_for_mtp_iteration reuse the first draft step's NSA/DSA topk indices across internal MTP iterations while keeping the unsafe topk>1 case disabled. Constraint: select_top_k_tokens can reorder hidden rows when topk > 1, so carried topk indices are only valid under topk == 1. Rejected: Enable reuse unconditionally | models without the config flag may not have compatible MTP index semantics. Rejected: Broaden to target-to-draft index reuse | separate semantic change with different correctness risks. Confidence: high Scope-risk: narrow Directive: Keep spec v1 and spec v2 MTP index reuse semantics aligned, including the topk==1 guard and per-draft-forward cleanup. Tested: python -m pytest test/registered/spec/eagle/test_eagle_v2_draft_extend_contract.py -q Tested: python -m py_compile python/sglang/srt/speculative/eagle_worker.py test/registered/spec/eagle/test_eagle_v2_draft_extend_contract.py Not-tested: full spec v1 GLM5 online throughput run
This commit is contained in:
@@ -156,6 +156,19 @@ class EAGLEWorker(TpModelWorker):
|
|||||||
memory_pool_config=target_worker.model_runner.memory_pool_config,
|
memory_pool_config=target_worker.model_runner.memory_pool_config,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Reuse the first draft step's NSA/DSA indexer topk across the rest of
|
||||||
|
# the MTP iteration when the model config says it is safe. The reuse is
|
||||||
|
# only valid for topk == 1: select_top_k_tokens reorders rows for topk
|
||||||
|
# > 1, which would desynchronize carried indices from hidden states.
|
||||||
|
self.index_share_for_mtp_iteration = (
|
||||||
|
getattr(
|
||||||
|
self.draft_model_runner.model_config.hf_config,
|
||||||
|
"index_share_for_mtp_iteration",
|
||||||
|
False,
|
||||||
|
)
|
||||||
|
and self.topk == 1
|
||||||
|
)
|
||||||
|
|
||||||
embed, head = self.target_worker.model_runner.model.get_embed_and_head()
|
embed, head = self.target_worker.model_runner.model.get_embed_and_head()
|
||||||
|
|
||||||
if self.speculative_algorithm.is_eagle3():
|
if self.speculative_algorithm.is_eagle3():
|
||||||
@@ -701,6 +714,10 @@ class EAGLEWorker(TpModelWorker):
|
|||||||
token_list: List[torch.Tensor] = []
|
token_list: List[torch.Tensor] = []
|
||||||
parents_list: List[torch.Tensor] = []
|
parents_list: List[torch.Tensor] = []
|
||||||
|
|
||||||
|
if self.index_share_for_mtp_iteration:
|
||||||
|
forward_batch.reuse_mtp_topk_indices = True
|
||||||
|
forward_batch.topk_indices = None
|
||||||
|
|
||||||
# Forward multiple steps
|
# Forward multiple steps
|
||||||
scores = None
|
scores = None
|
||||||
for i in range(self.speculative_num_steps):
|
for i in range(self.speculative_num_steps):
|
||||||
@@ -751,6 +768,10 @@ class EAGLEWorker(TpModelWorker):
|
|||||||
score_list, token_list, parents_list, self.speculative_num_draft_tokens
|
score_list, token_list, parents_list, self.speculative_num_draft_tokens
|
||||||
)
|
)
|
||||||
|
|
||||||
|
if self.index_share_for_mtp_iteration:
|
||||||
|
forward_batch.topk_indices = None
|
||||||
|
forward_batch.reuse_mtp_topk_indices = False
|
||||||
|
|
||||||
return parent_list, top_scores_index, draft_tokens
|
return parent_list, top_scores_index, draft_tokens
|
||||||
|
|
||||||
def clear_cache_pool(self):
|
def clear_cache_pool(self):
|
||||||
|
|||||||
@@ -365,6 +365,71 @@ def test_eagle_v2_draft_forward_scopes_mtp_index_reuse_to_one_draft_forward():
|
|||||||
assert topk_clears >= 2
|
assert topk_clears >= 2
|
||||||
|
|
||||||
|
|
||||||
|
def test_eagle_v1_draft_worker_enables_mtp_index_reuse_only_from_model_config():
|
||||||
|
"""Spec-v1 EAGLE must use the same guarded MTP index reuse contract."""
|
||||||
|
|
||||||
|
tree = _parse_module("python/sglang/srt/speculative/eagle_worker.py")
|
||||||
|
cls = _find_class(tree, "EAGLEWorker")
|
||||||
|
init = _find_method(cls, "__init__")
|
||||||
|
|
||||||
|
assert "index_share_for_mtp_iteration" in _assigned_self_attrs(init)
|
||||||
|
|
||||||
|
assign = next(
|
||||||
|
node
|
||||||
|
for node in ast.walk(init)
|
||||||
|
if isinstance(node, ast.Assign)
|
||||||
|
and any(
|
||||||
|
isinstance(target, ast.Attribute)
|
||||||
|
and isinstance(target.value, ast.Name)
|
||||||
|
and target.value.id == "self"
|
||||||
|
and target.attr == "index_share_for_mtp_iteration"
|
||||||
|
for target in node.targets
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
text = ast.unparse(assign.value)
|
||||||
|
assert "index_share_for_mtp_iteration" in text
|
||||||
|
assert "self.topk == 1" in text
|
||||||
|
|
||||||
|
|
||||||
|
def test_eagle_v1_draft_forward_scopes_mtp_index_reuse_to_one_draft_forward():
|
||||||
|
"""Spec-v1 EAGLE must clear transient MTP topk reuse state per call."""
|
||||||
|
|
||||||
|
tree = _parse_module("python/sglang/srt/speculative/eagle_worker.py")
|
||||||
|
cls = _find_class(tree, "EAGLEWorker")
|
||||||
|
func = _find_method(cls, "draft_forward")
|
||||||
|
|
||||||
|
reuse_assigns = []
|
||||||
|
topk_clears = 0
|
||||||
|
for node in ast.walk(func):
|
||||||
|
if not isinstance(node, ast.Assign):
|
||||||
|
continue
|
||||||
|
for target in node.targets:
|
||||||
|
if (
|
||||||
|
isinstance(target, ast.Attribute)
|
||||||
|
and isinstance(target.value, ast.Name)
|
||||||
|
and target.value.id == "forward_batch"
|
||||||
|
):
|
||||||
|
if target.attr == "reuse_mtp_topk_indices":
|
||||||
|
reuse_assigns.append(node.value)
|
||||||
|
if (
|
||||||
|
target.attr == "topk_indices"
|
||||||
|
and isinstance(node.value, ast.Constant)
|
||||||
|
and node.value.value is None
|
||||||
|
):
|
||||||
|
topk_clears += 1
|
||||||
|
|
||||||
|
assert any(
|
||||||
|
isinstance(value, ast.Constant) and value.value is True
|
||||||
|
for value in reuse_assigns
|
||||||
|
)
|
||||||
|
assert any(
|
||||||
|
isinstance(value, ast.Constant) and value.value is False
|
||||||
|
for value in reuse_assigns
|
||||||
|
)
|
||||||
|
assert topk_clears >= 2
|
||||||
|
|
||||||
|
|
||||||
def test_deepseek_nextn_reuses_and_updates_mtp_topk_indices_when_requested():
|
def test_deepseek_nextn_reuses_and_updates_mtp_topk_indices_when_requested():
|
||||||
"""NextN must pass cached MTP topk indices into the decoder and update them.
|
"""NextN must pass cached MTP topk indices into the decoder and update them.
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user