Unify spec v2's naming manner. (#15990)
This commit is contained in:
@@ -244,6 +244,7 @@ class Envs:
|
||||
# NPU
|
||||
SGLANG_NPU_DISABLE_ACL_FORMAT_WEIGHT = EnvBool(False)
|
||||
SGLANG_NPU_USE_MULTI_STREAM = EnvBool(False)
|
||||
SGLANG_NPU_USE_MLAPO = EnvBool(False)
|
||||
|
||||
# Quantization
|
||||
SGLANG_INT4_WEIGHT = EnvBool(False)
|
||||
|
||||
@@ -1854,16 +1854,16 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
|
||||
)
|
||||
|
||||
@property
|
||||
def is_eagle_v2(self):
|
||||
# FIXME: finally deprecate is_eagle_v2
|
||||
def is_spec_v2(self):
|
||||
# FIXME: finally deprecate is_spec_v2
|
||||
return self.enable_overlap and self.spec_algorithm.is_eagle()
|
||||
|
||||
def prepare_for_decode(self):
|
||||
self.forward_mode = ForwardMode.DECODE
|
||||
bs = len(self.reqs)
|
||||
|
||||
if self.is_eagle_v2:
|
||||
# TODO(spec-v2): all v2 spec should go through this path
|
||||
if self.is_spec_v2:
|
||||
# TODO(spec-v2): all spec v2 should go through this path
|
||||
draft_input: EagleDraftInput = self.spec_info
|
||||
draft_input.prepare_for_decode(self)
|
||||
|
||||
@@ -1942,7 +1942,7 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
|
||||
)
|
||||
|
||||
def maybe_wait_verify_done(self):
|
||||
if self.is_eagle_v2:
|
||||
if self.is_spec_v2:
|
||||
draft_input: EagleDraftInput = self.spec_info
|
||||
if draft_input.verify_done is not None:
|
||||
draft_input.verify_done.synchronize()
|
||||
@@ -2018,7 +2018,7 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
|
||||
# NOTE: spec_info filtered before batch filtering only happens in:
|
||||
# - Spec v1's verify phase
|
||||
# - Only for decode batch (running_batch)
|
||||
has_been_filtered = v1_spec_info_filtered and not self.is_eagle_v2
|
||||
has_been_filtered = v1_spec_info_filtered and not self.is_spec_v2
|
||||
|
||||
if self.spec_info:
|
||||
self.spec_info.filter_batch(
|
||||
@@ -2027,7 +2027,7 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
|
||||
)
|
||||
|
||||
def merge_batch(self, other: "ScheduleBatch"):
|
||||
# NOTE: in v2 eagle mode, we do not need wait verify here because
|
||||
# NOTE: in spec v2 mode, we do not need wait verify here because
|
||||
# 1) current batch is always prefill, whose seq_lens is not a future
|
||||
# 2) other batch is always decode, which is finished in previous step
|
||||
|
||||
|
||||
@@ -1165,7 +1165,7 @@ class Scheduler(
|
||||
# TODO(lsyin): support overlap + spec + grammar
|
||||
need_grammar_sync = (
|
||||
batch
|
||||
and batch.is_eagle_v2
|
||||
and batch.is_spec_v2
|
||||
and batch.has_grammar
|
||||
and batch.forward_mode.is_decode()
|
||||
and len(self.result_queue) > 0
|
||||
@@ -2225,8 +2225,8 @@ class Scheduler(
|
||||
# FIXME(lsyin): move this assignment elsewhere
|
||||
future_indices_or_next_token_ids = -future_indices.indices
|
||||
|
||||
if batch.is_eagle_v2:
|
||||
# FIXME(lsyin): tmp code for eagle v2
|
||||
if batch.is_spec_v2:
|
||||
# FIXME(lsyin): tmp code for spec v2
|
||||
# We only keep future indices for next draft input
|
||||
|
||||
batch.spec_info = batch_result.next_draft_input
|
||||
|
||||
@@ -359,7 +359,7 @@ class SchedulerOutputProcessorMixin:
|
||||
next_token_ids = next_token_ids.tolist()
|
||||
if batch.return_logprob:
|
||||
next_token_logprobs = logits_output.next_token_logprobs.tolist()
|
||||
elif batch.is_eagle_v2:
|
||||
elif batch.is_spec_v2:
|
||||
next_token_ids = self._resolve_spec_overlap_token_ids(result, batch)
|
||||
|
||||
self.num_generated_tokens += len(batch.reqs)
|
||||
@@ -386,8 +386,8 @@ class SchedulerOutputProcessorMixin:
|
||||
new_accepted_len = 1
|
||||
if batch.spec_algorithm.is_none():
|
||||
req.output_ids.append(next_token_id)
|
||||
elif batch.is_eagle_v2:
|
||||
# Only v2 eagle's output_ids are updated here.
|
||||
elif batch.is_spec_v2:
|
||||
# Only spec v2's output_ids are updated here.
|
||||
req.output_ids.extend(next_token_id)
|
||||
new_accepted_len = len(next_token_id)
|
||||
|
||||
@@ -438,7 +438,7 @@ class SchedulerOutputProcessorMixin:
|
||||
if batch.spec_algorithm.is_none():
|
||||
# Normal decode: single token
|
||||
req.grammar.accept_token(next_token_id)
|
||||
elif batch.is_eagle_v2:
|
||||
elif batch.is_spec_v2:
|
||||
# Speculative decode: next_token_id is a list of accepted tokens
|
||||
for token_id in next_token_id:
|
||||
req.grammar.accept_token(token_id)
|
||||
|
||||
@@ -141,7 +141,7 @@ class ForwardMode(IntEnum):
|
||||
)
|
||||
|
||||
def is_draft_extend_v2(self):
|
||||
# For fixed shape logits output in v2 eagle worker
|
||||
# For fixed shape logits output in eagle v2 worker
|
||||
return self == ForwardMode.DRAFT_EXTEND_V2
|
||||
|
||||
def is_extend_or_draft_extend_or_mixed(self, include_draft_extend_v2: bool = False):
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
import os
|
||||
import unittest
|
||||
from types import SimpleNamespace
|
||||
|
||||
import requests
|
||||
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.utils import kill_process_tree
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k
|
||||
@@ -26,7 +26,6 @@ SERVER_LAUNCH_TIMEOUT = 1200
|
||||
class TestDeepseekV3FP4MTP(CustomTestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
os.environ["SGLANG_ENABLE_SPEC_V2"] = "1"
|
||||
cls.model = FULL_DEEPSEEK_V3_FP4_MODEL_PATH
|
||||
cls.base_url = DEFAULT_URL_FOR_TEST
|
||||
other_args = [
|
||||
@@ -51,18 +50,17 @@ class TestDeepseekV3FP4MTP(CustomTestCase):
|
||||
"--model-loader-extra-config",
|
||||
'{"enable_multithread_load": true,"num_threads": 64}',
|
||||
]
|
||||
cls.process = popen_launch_server(
|
||||
cls.model,
|
||||
cls.base_url,
|
||||
timeout=SERVER_LAUNCH_TIMEOUT,
|
||||
other_args=other_args,
|
||||
)
|
||||
with envs.SGLANG_ENABLE_SPEC_V2.override(True):
|
||||
cls.process = popen_launch_server(
|
||||
cls.model,
|
||||
cls.base_url,
|
||||
timeout=SERVER_LAUNCH_TIMEOUT,
|
||||
other_args=other_args,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
kill_process_tree(cls.process.pid)
|
||||
if "SGLANG_ENABLE_SPEC_V2" in os.environ:
|
||||
del os.environ["SGLANG_ENABLE_SPEC_V2"]
|
||||
|
||||
def test_a_gsm8k(
|
||||
self,
|
||||
|
||||
@@ -30,7 +30,7 @@ class TestEagleConstrainedDecoding(
|
||||
model = DEFAULT_TARGET_MODEL_EAGLE
|
||||
draft_model = DEFAULT_DRAFT_MODEL_EAGLE
|
||||
grammar_backend = "xgrammar"
|
||||
eagle_v2 = False
|
||||
spec_v2 = False
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
@@ -59,7 +59,7 @@ class TestEagleConstrainedDecoding(
|
||||
cls.grammar_backend,
|
||||
]
|
||||
launch_args.extend(cls.other_launch_args)
|
||||
with envs.SGLANG_ENABLE_SPEC_V2.override(cls.eagle_v2):
|
||||
with envs.SGLANG_ENABLE_SPEC_V2.override(cls.spec_v2):
|
||||
cls.process = popen_launch_server(
|
||||
cls.model,
|
||||
cls.base_url,
|
||||
@@ -73,7 +73,7 @@ class TestEagleConstrainedDecoding(
|
||||
|
||||
|
||||
class TestEagleConstrainedDecodingV2(TestEagleConstrainedDecoding):
|
||||
eagle_v2 = True
|
||||
spec_v2 = True
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
import os
|
||||
import unittest
|
||||
from types import SimpleNamespace
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.utils import kill_process_tree
|
||||
from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k
|
||||
from sglang.test.test_utils import (
|
||||
@@ -54,12 +54,9 @@ class TestAscendDeepSeekMTP(CustomTestCase):
|
||||
2,
|
||||
]
|
||||
|
||||
cls.extra_envs = {
|
||||
"SGLANG_NPU_USE_MLAPO": "1",
|
||||
"SGLANG_ENABLE_SPEC_V2": "1",
|
||||
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
|
||||
}
|
||||
os.environ.update(cls.extra_envs)
|
||||
envs.SGLANG_NPU_USE_MLAPO.set(True)
|
||||
envs.SGLANG_ENABLE_SPEC_V2.set(True)
|
||||
envs.SGLANG_ENABLE_OVERLAP_PLAN_STREAM.set(True)
|
||||
|
||||
def test_a_gsm8k(self):
|
||||
for model in self.models:
|
||||
|
||||
Reference in New Issue
Block a user