feat(SpecEagleV2): add standalone_worker_v2 (#12625)
Co-authored-by: liugaoji.lgj <liugaoji.lgj@alibaba-inc.com>
This commit is contained in:
co-authored by
liugaoji.lgj
parent
b6871ba7c9
commit
7518dc3532
@@ -499,7 +499,7 @@ class Scheduler(
|
||||
|
||||
# Draft workers are looked up via `SpeculativeAlgorithm` registry; new
|
||||
# algorithms should register their factory instead of patching this code.
|
||||
if self.spec_algorithm.is_eagle():
|
||||
if self.spec_algorithm.supports_spec_v2():
|
||||
draft_worker_kwargs["enable_overlap"] = self.enable_overlap
|
||||
|
||||
# FIXME: refactor the draft worker registration logic
|
||||
@@ -852,7 +852,7 @@ class Scheduler(
|
||||
|
||||
if self.draft_worker is None or self.spec_algorithm.is_ngram():
|
||||
draft_token_to_kv_pool = None
|
||||
elif self.spec_algorithm.is_eagle() and self.enable_overlap:
|
||||
elif self.spec_algorithm.supports_spec_v2() and self.enable_overlap:
|
||||
if self.server_args.enable_multi_layer_eagle:
|
||||
draft_runner = self.draft_worker.draft_worker.draft_runner_list[0]
|
||||
else:
|
||||
@@ -930,11 +930,13 @@ class Scheduler(
|
||||
hidden_size=(
|
||||
model_config.hidden_size
|
||||
if self.spec_algorithm.is_eagle()
|
||||
or self.spec_algorithm.is_standalone()
|
||||
else 16 # minimal padding size for RDMA
|
||||
),
|
||||
hidden_states_dtype=(
|
||||
model_config.dtype
|
||||
if self.spec_algorithm.is_eagle()
|
||||
or self.spec_algorithm.is_standalone()
|
||||
else torch.float32
|
||||
),
|
||||
custom_mem_pool=self.token_to_kv_pool_allocator.get_kvcache().maybe_get_custom_mem_pool(),
|
||||
|
||||
Reference in New Issue
Block a user