feat(SpecEagleV2): add standalone_worker_v2 (#12625)

Co-authored-by: liugaoji.lgj <liugaoji.lgj@alibaba-inc.com>
This commit is contained in:
Gaoji Liu
2025-12-30 17:55:04 +08:00
committed by GitHub
co-authored by liugaoji.lgj
parent b6871ba7c9
commit 7518dc3532
14 changed files with 317 additions and 17 deletions
+4 -2
View File
@@ -499,7 +499,7 @@ class Scheduler(
# Draft workers are looked up via `SpeculativeAlgorithm` registry; new
# algorithms should register their factory instead of patching this code.
if self.spec_algorithm.is_eagle():
if self.spec_algorithm.supports_spec_v2():
draft_worker_kwargs["enable_overlap"] = self.enable_overlap
# FIXME: refactor the draft worker registration logic
@@ -852,7 +852,7 @@ class Scheduler(
if self.draft_worker is None or self.spec_algorithm.is_ngram():
draft_token_to_kv_pool = None
elif self.spec_algorithm.is_eagle() and self.enable_overlap:
elif self.spec_algorithm.supports_spec_v2() and self.enable_overlap:
if self.server_args.enable_multi_layer_eagle:
draft_runner = self.draft_worker.draft_worker.draft_runner_list[0]
else:
@@ -930,11 +930,13 @@ class Scheduler(
hidden_size=(
model_config.hidden_size
if self.spec_algorithm.is_eagle()
or self.spec_algorithm.is_standalone()
else 16 # minimal padding size for RDMA
),
hidden_states_dtype=(
model_config.dtype
if self.spec_algorithm.is_eagle()
or self.spec_algorithm.is_standalone()
else torch.float32
),
custom_mem_pool=self.token_to_kv_pool_allocator.get_kvcache().maybe_get_custom_mem_pool(),