Beta spec-overlap for EAGLE (#11398)
Co-authored-by: Lianmin Zheng <15100009+merrymercy@users.noreply.github.com> Co-authored-by: Hanming Lu <69857889+hanming-lu@users.noreply.github.com>
This commit is contained in:
@@ -312,6 +312,7 @@ class ServerArgs:
|
||||
nsa_decode: str = "fa3"
|
||||
|
||||
# Speculative decoding
|
||||
enable_beta_spec: bool = False
|
||||
speculative_algorithm: Optional[str] = None
|
||||
speculative_draft_model_path: Optional[str] = None
|
||||
speculative_draft_model_revision: Optional[str] = None
|
||||
@@ -1103,11 +1104,19 @@ class ServerArgs:
|
||||
)
|
||||
if self.max_running_requests is None:
|
||||
self.max_running_requests = 48
|
||||
self.disable_overlap_schedule = True
|
||||
logger.warning(
|
||||
"Overlap scheduler is disabled because of using "
|
||||
"eagle speculative decoding."
|
||||
)
|
||||
|
||||
if self.speculative_algorithm == "EAGLE" and self.enable_beta_spec:
|
||||
self.disable_overlap_schedule = False
|
||||
logger.warning(
|
||||
"Beta spec is enabled for eagle speculative decoding and overlap schedule is turned on."
|
||||
)
|
||||
|
||||
if not self.enable_beta_spec:
|
||||
self.disable_overlap_schedule = True
|
||||
logger.warning(
|
||||
"Overlap scheduler is disabled because of using eagle3 and standalone speculative decoding."
|
||||
)
|
||||
|
||||
if self.enable_mixed_chunk:
|
||||
self.enable_mixed_chunk = False
|
||||
logger.warning(
|
||||
@@ -2127,6 +2136,7 @@ class ServerArgs:
|
||||
)
|
||||
|
||||
# Speculative decoding
|
||||
parser.add_argument("--enable-beta-spec", action="store_true")
|
||||
parser.add_argument(
|
||||
"--speculative-algorithm",
|
||||
type=str,
|
||||
|
||||
Reference in New Issue
Block a user