diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py index dc12cb37e..870af8b6c 100644 --- a/python/sglang/srt/environ.py +++ b/python/sglang/srt/environ.py @@ -244,6 +244,7 @@ class Envs: # NPU SGLANG_NPU_DISABLE_ACL_FORMAT_WEIGHT = EnvBool(False) SGLANG_NPU_USE_MULTI_STREAM = EnvBool(False) + SGLANG_NPU_USE_MLAPO = EnvBool(False) # Quantization SGLANG_INT4_WEIGHT = EnvBool(False) diff --git a/python/sglang/srt/managers/schedule_batch.py b/python/sglang/srt/managers/schedule_batch.py index dbf761807..9de5da95d 100644 --- a/python/sglang/srt/managers/schedule_batch.py +++ b/python/sglang/srt/managers/schedule_batch.py @@ -1854,16 +1854,16 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin): ) @property - def is_eagle_v2(self): - # FIXME: finally deprecate is_eagle_v2 + def is_spec_v2(self): + # FIXME: finally deprecate is_spec_v2 return self.enable_overlap and self.spec_algorithm.is_eagle() def prepare_for_decode(self): self.forward_mode = ForwardMode.DECODE bs = len(self.reqs) - if self.is_eagle_v2: - # TODO(spec-v2): all v2 spec should go through this path + if self.is_spec_v2: + # TODO(spec-v2): all spec v2 should go through this path draft_input: EagleDraftInput = self.spec_info draft_input.prepare_for_decode(self) @@ -1942,7 +1942,7 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin): ) def maybe_wait_verify_done(self): - if self.is_eagle_v2: + if self.is_spec_v2: draft_input: EagleDraftInput = self.spec_info if draft_input.verify_done is not None: draft_input.verify_done.synchronize() @@ -2018,7 +2018,7 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin): # NOTE: spec_info filtered before batch filtering only happens in: # - Spec v1's verify phase # - Only for decode batch (running_batch) - has_been_filtered = v1_spec_info_filtered and not self.is_eagle_v2 + has_been_filtered = v1_spec_info_filtered and not self.is_spec_v2 if self.spec_info: self.spec_info.filter_batch( @@ -2027,7 +2027,7 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin): ) def merge_batch(self, other: "ScheduleBatch"): - # NOTE: in v2 eagle mode, we do not need wait verify here because + # NOTE: in spec v2 mode, we do not need wait verify here because # 1) current batch is always prefill, whose seq_lens is not a future # 2) other batch is always decode, which is finished in previous step diff --git a/python/sglang/srt/managers/scheduler.py b/python/sglang/srt/managers/scheduler.py index 9d1f0450f..975a20250 100644 --- a/python/sglang/srt/managers/scheduler.py +++ b/python/sglang/srt/managers/scheduler.py @@ -1165,7 +1165,7 @@ class Scheduler( # TODO(lsyin): support overlap + spec + grammar need_grammar_sync = ( batch - and batch.is_eagle_v2 + and batch.is_spec_v2 and batch.has_grammar and batch.forward_mode.is_decode() and len(self.result_queue) > 0 @@ -2225,8 +2225,8 @@ class Scheduler( # FIXME(lsyin): move this assignment elsewhere future_indices_or_next_token_ids = -future_indices.indices - if batch.is_eagle_v2: - # FIXME(lsyin): tmp code for eagle v2 + if batch.is_spec_v2: + # FIXME(lsyin): tmp code for spec v2 # We only keep future indices for next draft input batch.spec_info = batch_result.next_draft_input diff --git a/python/sglang/srt/managers/scheduler_output_processor_mixin.py b/python/sglang/srt/managers/scheduler_output_processor_mixin.py index a03c8c279..c97bd8243 100644 --- a/python/sglang/srt/managers/scheduler_output_processor_mixin.py +++ b/python/sglang/srt/managers/scheduler_output_processor_mixin.py @@ -359,7 +359,7 @@ class SchedulerOutputProcessorMixin: next_token_ids = next_token_ids.tolist() if batch.return_logprob: next_token_logprobs = logits_output.next_token_logprobs.tolist() - elif batch.is_eagle_v2: + elif batch.is_spec_v2: next_token_ids = self._resolve_spec_overlap_token_ids(result, batch) self.num_generated_tokens += len(batch.reqs) @@ -386,8 +386,8 @@ class SchedulerOutputProcessorMixin: new_accepted_len = 1 if batch.spec_algorithm.is_none(): req.output_ids.append(next_token_id) - elif batch.is_eagle_v2: - # Only v2 eagle's output_ids are updated here. + elif batch.is_spec_v2: + # Only spec v2's output_ids are updated here. req.output_ids.extend(next_token_id) new_accepted_len = len(next_token_id) @@ -438,7 +438,7 @@ class SchedulerOutputProcessorMixin: if batch.spec_algorithm.is_none(): # Normal decode: single token req.grammar.accept_token(next_token_id) - elif batch.is_eagle_v2: + elif batch.is_spec_v2: # Speculative decode: next_token_id is a list of accepted tokens for token_id in next_token_id: req.grammar.accept_token(token_id) diff --git a/python/sglang/srt/model_executor/forward_batch_info.py b/python/sglang/srt/model_executor/forward_batch_info.py index df4081bb5..e71b71b83 100644 --- a/python/sglang/srt/model_executor/forward_batch_info.py +++ b/python/sglang/srt/model_executor/forward_batch_info.py @@ -141,7 +141,7 @@ class ForwardMode(IntEnum): ) def is_draft_extend_v2(self): - # For fixed shape logits output in v2 eagle worker + # For fixed shape logits output in eagle v2 worker return self == ForwardMode.DRAFT_EXTEND_V2 def is_extend_or_draft_extend_or_mixed(self, include_draft_extend_v2: bool = False): diff --git a/test/registered/spec/eagle/test_deepseek_v3_fp4_mtp_stage_b.py b/test/registered/spec/eagle/test_deepseek_v3_fp4_mtp_stage_b.py index 2da84a77b..497425036 100644 --- a/test/registered/spec/eagle/test_deepseek_v3_fp4_mtp_stage_b.py +++ b/test/registered/spec/eagle/test_deepseek_v3_fp4_mtp_stage_b.py @@ -1,9 +1,9 @@ -import os import unittest from types import SimpleNamespace import requests +from sglang.srt.environ import envs from sglang.srt.utils import kill_process_tree from sglang.test.ci.ci_register import register_cuda_ci from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k @@ -26,7 +26,6 @@ SERVER_LAUNCH_TIMEOUT = 1200 class TestDeepseekV3FP4MTP(CustomTestCase): @classmethod def setUpClass(cls): - os.environ["SGLANG_ENABLE_SPEC_V2"] = "1" cls.model = FULL_DEEPSEEK_V3_FP4_MODEL_PATH cls.base_url = DEFAULT_URL_FOR_TEST other_args = [ @@ -51,18 +50,17 @@ class TestDeepseekV3FP4MTP(CustomTestCase): "--model-loader-extra-config", '{"enable_multithread_load": true,"num_threads": 64}', ] - cls.process = popen_launch_server( - cls.model, - cls.base_url, - timeout=SERVER_LAUNCH_TIMEOUT, - other_args=other_args, - ) + with envs.SGLANG_ENABLE_SPEC_V2.override(True): + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=SERVER_LAUNCH_TIMEOUT, + other_args=other_args, + ) @classmethod def tearDownClass(cls): kill_process_tree(cls.process.pid) - if "SGLANG_ENABLE_SPEC_V2" in os.environ: - del os.environ["SGLANG_ENABLE_SPEC_V2"] def test_a_gsm8k( self, diff --git a/test/registered/spec/eagle/test_eagle_constrained_decoding.py b/test/registered/spec/eagle/test_eagle_constrained_decoding.py index 76b72b893..0ebbd96e0 100644 --- a/test/registered/spec/eagle/test_eagle_constrained_decoding.py +++ b/test/registered/spec/eagle/test_eagle_constrained_decoding.py @@ -30,7 +30,7 @@ class TestEagleConstrainedDecoding( model = DEFAULT_TARGET_MODEL_EAGLE draft_model = DEFAULT_DRAFT_MODEL_EAGLE grammar_backend = "xgrammar" - eagle_v2 = False + spec_v2 = False @classmethod def setUpClass(cls): @@ -59,7 +59,7 @@ class TestEagleConstrainedDecoding( cls.grammar_backend, ] launch_args.extend(cls.other_launch_args) - with envs.SGLANG_ENABLE_SPEC_V2.override(cls.eagle_v2): + with envs.SGLANG_ENABLE_SPEC_V2.override(cls.spec_v2): cls.process = popen_launch_server( cls.model, cls.base_url, @@ -73,7 +73,7 @@ class TestEagleConstrainedDecoding( class TestEagleConstrainedDecodingV2(TestEagleConstrainedDecoding): - eagle_v2 = True + spec_v2 = True if __name__ == "__main__": diff --git a/test/srt/ascend/test_ascend_deepseek_mtp.py b/test/srt/ascend/test_ascend_deepseek_mtp.py index 43089f885..a04217fac 100644 --- a/test/srt/ascend/test_ascend_deepseek_mtp.py +++ b/test/srt/ascend/test_ascend_deepseek_mtp.py @@ -1,8 +1,8 @@ -import os import unittest from types import SimpleNamespace from urllib.parse import urlparse +from sglang.srt.environ import envs from sglang.srt.utils import kill_process_tree from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k from sglang.test.test_utils import ( @@ -54,12 +54,9 @@ class TestAscendDeepSeekMTP(CustomTestCase): 2, ] - cls.extra_envs = { - "SGLANG_NPU_USE_MLAPO": "1", - "SGLANG_ENABLE_SPEC_V2": "1", - "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1", - } - os.environ.update(cls.extra_envs) + envs.SGLANG_NPU_USE_MLAPO.set(True) + envs.SGLANG_ENABLE_SPEC_V2.set(True) + envs.SGLANG_ENABLE_OVERLAP_PLAN_STREAM.set(True) def test_a_gsm8k(self): for model in self.models: