Beta spec-overlap for EAGLE (#11398)

Co-authored-by: Lianmin Zheng <15100009+merrymercy@users.noreply.github.com>
Co-authored-by: Hanming Lu <69857889+hanming-lu@users.noreply.github.com>
This commit is contained in:
Liangsheng Yin
2025-10-12 11:02:22 +08:00
committed by GitHub
parent 47c606d3dc
commit 20a6c0a63d
21 changed files with 1567 additions and 108 deletions

View File

@@ -312,6 +312,7 @@ class ServerArgs:
nsa_decode: str = "fa3"
# Speculative decoding
enable_beta_spec: bool = False
speculative_algorithm: Optional[str] = None
speculative_draft_model_path: Optional[str] = None
speculative_draft_model_revision: Optional[str] = None
@@ -1103,11 +1104,19 @@ class ServerArgs:
)
if self.max_running_requests is None:
self.max_running_requests = 48
self.disable_overlap_schedule = True
logger.warning(
"Overlap scheduler is disabled because of using "
"eagle speculative decoding."
)
if self.speculative_algorithm == "EAGLE" and self.enable_beta_spec:
self.disable_overlap_schedule = False
logger.warning(
"Beta spec is enabled for eagle speculative decoding and overlap schedule is turned on."
)
if not self.enable_beta_spec:
self.disable_overlap_schedule = True
logger.warning(
"Overlap scheduler is disabled because of using eagle3 and standalone speculative decoding."
)
if self.enable_mixed_chunk:
self.enable_mixed_chunk = False
logger.warning(
@@ -2127,6 +2136,7 @@ class ServerArgs:
)
# Speculative decoding
parser.add_argument("--enable-beta-spec", action="store_true")
parser.add_argument(
"--speculative-algorithm",
type=str,