From c5e363e8e0ea7c1fe9873b951f556979fb949196 Mon Sep 17 00:00:00 2001 From: Alison Shao <54658187+alisonshao@users.noreply.github.com> Date: Tue, 13 Jan 2026 20:18:42 -0800 Subject: [PATCH] test: split Qwen3 Next tests and disable PCG tests due to intermittent failures (#16989) --- test/srt/models/test_qwen3_next_models.py | 212 ------------------ test/srt/models/test_qwen3_next_models_mtp.py | 212 ++++++++++++++++++ test/srt/models/test_qwen3_next_models_pcg.py | 70 ++++++ test/srt/run_suite.py | 6 +- 4 files changed, 287 insertions(+), 213 deletions(-) create mode 100644 test/srt/models/test_qwen3_next_models_mtp.py create mode 100644 test/srt/models/test_qwen3_next_models_pcg.py diff --git a/test/srt/models/test_qwen3_next_models.py b/test/srt/models/test_qwen3_next_models.py index f8eff84cb..3da79a3d2 100644 --- a/test/srt/models/test_qwen3_next_models.py +++ b/test/srt/models/test_qwen3_next_models.py @@ -22,11 +22,6 @@ ACC_THRESHOLDS = { QWEN3_NEXT_MODEL: {"kl_div": 0.0025, "gsm8k": 0.93}, } -# MTP has higher KL divergence threshold -ACC_THRESHOLDS_MTP = { - QWEN3_NEXT_MODEL: {"kl_div": 0.008, "gsm8k": 0.93}, -} - def send_request_helper(base_url: str, text: str): response = requests.post( @@ -128,212 +123,5 @@ class TestQwen3Next(CustomTestCase): print("test_prefix_cache_branching passed") -class TestQwen3NextMTP(CustomTestCase): - @classmethod - def setUpClass(cls): - cls.model = QWEN3_NEXT_MODEL - cls.base_url = DEFAULT_URL_FOR_TEST - cls.process = popen_launch_server( - cls.model, - cls.base_url, - timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - other_args=[ - "--trust-remote-code", - "--speculative-algorithm", - "NEXTN", - "--speculative-num-steps", - "3", - "--speculative-eagle-topk", - "1", - "--speculative-num-draft-tokens", - "4", - "--mem-fraction-static", - "0.8", - "--tp", - "4", - "--chunked-prefill-size", - "2048", - "--mamba-scheduler-strategy", - "no_buffer", - ], - ) - - @classmethod - def tearDownClass(cls): - kill_process_tree(cls.process.pid) - - def test_gsm8k(self): - args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=200, - max_new_tokens=512, - parallel=128, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), - ) - metrics = run_eval(args) - print(f"{metrics=}") - self.assertGreaterEqual( - metrics["accuracy"], ACC_THRESHOLDS[self.model]["gsm8k"] - ) - - def test_input_output_logprobs_match_prefill_cache_hit(self): - test_input_output_logprobs_match_prefill_cache_hit_helper( - self.base_url, - ACC_THRESHOLDS, - self.model, - max_samples=32, - max_new_tokens=512, - ) - - def test_input_output_logprobs_match_decode_cache_hit(self): - test_input_output_logprobs_match_decode_cache_hit_helper( - self.base_url, - ACC_THRESHOLDS, - self.model, - max_samples=32, - max_new_tokens=512, - ) - - -class TestQwen3NextMTPTopk(CustomTestCase): - @classmethod - def setUpClass(cls): - cls.model = QWEN3_NEXT_MODEL - cls.base_url = DEFAULT_URL_FOR_TEST - cls.process = popen_launch_server( - cls.model, - cls.base_url, - timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - other_args=[ - "--trust-remote-code", - "--speculative-algorithm", - "NEXTN", - "--speculative-num-steps", - "5", - "--speculative-eagle-topk", - "4", - "--speculative-num-draft-tokens", - "8", - "--mem-fraction-static", - "0.8", - "--tp", - "4", - "--chunked-prefill-size", - "2048", - "--mamba-scheduler-strategy", - "extra_buffer", - "--mamba-track-interval", - "128", - ], - ) - - @classmethod - def tearDownClass(cls): - kill_process_tree(cls.process.pid) - - def test_gsm8k(self): - args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=200, - max_new_tokens=512, - parallel=128, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), - ) - metrics = run_eval(args) - print(f"{metrics=}") - self.assertGreaterEqual( - metrics["accuracy"], ACC_THRESHOLDS_MTP[self.model]["gsm8k"] - ) - - def test_input_output_logprobs_match_prefill_cache_hit(self): - test_input_output_logprobs_match_prefill_cache_hit_helper( - self.base_url, - ACC_THRESHOLDS_MTP, - self.model, - max_samples=32, - max_new_tokens=512, - ) - - def test_input_output_logprobs_match_decode_cache_hit(self): - test_input_output_logprobs_match_decode_cache_hit_helper( - self.base_url, - ACC_THRESHOLDS_MTP, - self.model, - max_samples=32, - max_new_tokens=512, - ) - - def test_prefix_cache_branching(self): - print("running test_prefix_cache_branching") - requests.get(self.base_url + "/flush_cache") - branching_pos = 257 - text_prefix = "hi" * branching_pos - suffix_list = ["this" * 256, "here" * 256, "that" * 256] - cache_hit_list = [False, False, True] - - # First request only prefill the entire sequence - # Second request won't have cache hit, but will cache the branching point - # Third request will have cache hit on the branching point - for i, (suffix, cache_hit) in enumerate( - zip(suffix_list, cache_hit_list, strict=True) - ): - result = send_request_helper(self.base_url, text_prefix + suffix) - cached_tokens = result["meta_info"]["cached_tokens"] - if cache_hit: - expected_cached_tokens = branching_pos // 64 * 64 - assert ( - cached_tokens == expected_cached_tokens - ), f"{i=}, {cache_hit=}, {cached_tokens=} is not equal to {expected_cached_tokens=}, {branching_pos=}" - else: - assert ( - cached_tokens == 0 - ), f"{i=}, {cache_hit=}, {cached_tokens=} is not 0" - print("test_prefix_cache_branching passed") - - -class TestQwen3NextPiecewiseCudaGraph(CustomTestCase): - - @classmethod - def setUpClass(cls): - cls.model = QWEN3_NEXT_MODEL - cls.base_url = DEFAULT_URL_FOR_TEST - cls.process = popen_launch_server( - cls.model, - cls.base_url, - timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - other_args=[ - "--tp", - "4", - "--enable-piecewise-cuda-graph", - "--piecewise-cuda-graph-compiler", - "eager", - ], - ) - - @classmethod - def tearDownClass(cls): - kill_process_tree(cls.process.pid) - - def test_gsm8k(self): - args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=200, - max_new_tokens=512, - parallel=128, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), - ) - metrics = run_eval(args) - print(f"{metrics=}") - self.assertGreaterEqual( - metrics["accuracy"], ACC_THRESHOLDS[self.model]["gsm8k"] - ) - - if __name__ == "__main__": unittest.main() diff --git a/test/srt/models/test_qwen3_next_models_mtp.py b/test/srt/models/test_qwen3_next_models_mtp.py new file mode 100644 index 000000000..ecbc1337e --- /dev/null +++ b/test/srt/models/test_qwen3_next_models_mtp.py @@ -0,0 +1,212 @@ +import unittest +from types import SimpleNamespace + +import requests + +from sglang.srt.utils import kill_process_tree +from sglang.test.few_shot_gsm8k import run_eval +from sglang.test.kl_test_utils import ( + test_input_output_logprobs_match_decode_cache_hit_helper, + test_input_output_logprobs_match_prefill_cache_hit_helper, +) +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + +QWEN3_NEXT_MODEL = "Qwen/Qwen3-Next-80B-A3B-Instruct" + +ACC_THRESHOLDS = { + QWEN3_NEXT_MODEL: {"kl_div": 0.0025, "gsm8k": 0.93}, +} + +# MTP has higher KL divergence threshold +ACC_THRESHOLDS_MTP = { + QWEN3_NEXT_MODEL: {"kl_div": 0.008, "gsm8k": 0.93}, +} + + +def send_request_helper(base_url: str, text: str): + response = requests.post( + base_url + "/generate", + json={ + "text": text, + "sampling_params": { + "max_new_tokens": 1, + }, + }, + ) + return response.json() + + +class TestQwen3NextMTP(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = QWEN3_NEXT_MODEL + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--speculative-algorithm", + "NEXTN", + "--speculative-num-steps", + "3", + "--speculative-eagle-topk", + "1", + "--speculative-num-draft-tokens", + "4", + "--mem-fraction-static", + "0.8", + "--tp", + "4", + "--chunked-prefill-size", + "2048", + "--mamba-scheduler-strategy", + "no_buffer", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval(args) + print(f"{metrics=}") + self.assertGreaterEqual( + metrics["accuracy"], ACC_THRESHOLDS[self.model]["gsm8k"] + ) + + def test_input_output_logprobs_match_prefill_cache_hit(self): + test_input_output_logprobs_match_prefill_cache_hit_helper( + self.base_url, + ACC_THRESHOLDS, + self.model, + max_samples=32, + max_new_tokens=512, + ) + + def test_input_output_logprobs_match_decode_cache_hit(self): + test_input_output_logprobs_match_decode_cache_hit_helper( + self.base_url, + ACC_THRESHOLDS, + self.model, + max_samples=32, + max_new_tokens=512, + ) + + +class TestQwen3NextMTPTopk(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = QWEN3_NEXT_MODEL + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--speculative-algorithm", + "NEXTN", + "--speculative-num-steps", + "5", + "--speculative-eagle-topk", + "4", + "--speculative-num-draft-tokens", + "8", + "--mem-fraction-static", + "0.8", + "--tp", + "4", + "--chunked-prefill-size", + "2048", + "--mamba-scheduler-strategy", + "extra_buffer", + "--mamba-track-interval", + "128", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval(args) + print(f"{metrics=}") + self.assertGreaterEqual( + metrics["accuracy"], ACC_THRESHOLDS_MTP[self.model]["gsm8k"] + ) + + def test_input_output_logprobs_match_prefill_cache_hit(self): + test_input_output_logprobs_match_prefill_cache_hit_helper( + self.base_url, + ACC_THRESHOLDS_MTP, + self.model, + max_samples=32, + max_new_tokens=512, + ) + + def test_input_output_logprobs_match_decode_cache_hit(self): + test_input_output_logprobs_match_decode_cache_hit_helper( + self.base_url, + ACC_THRESHOLDS_MTP, + self.model, + max_samples=32, + max_new_tokens=512, + ) + + def test_prefix_cache_branching(self): + print("running test_prefix_cache_branching") + requests.get(self.base_url + "/flush_cache") + branching_pos = 257 + text_prefix = "hi" * branching_pos + suffix_list = ["this" * 256, "here" * 256, "that" * 256] + cache_hit_list = [False, False, True] + + # First request only prefill the entire sequence + # Second request won't have cache hit, but will cache the branching point + # Third request will have cache hit on the branching point + for i, (suffix, cache_hit) in enumerate( + zip(suffix_list, cache_hit_list, strict=True) + ): + result = send_request_helper(self.base_url, text_prefix + suffix) + cached_tokens = result["meta_info"]["cached_tokens"] + if cache_hit: + expected_cached_tokens = branching_pos // 64 * 64 + assert ( + cached_tokens == expected_cached_tokens + ), f"{i=}, {cache_hit=}, {cached_tokens=} is not equal to {expected_cached_tokens=}, {branching_pos=}" + else: + assert ( + cached_tokens == 0 + ), f"{i=}, {cache_hit=}, {cached_tokens=} is not 0" + print("test_prefix_cache_branching passed") + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/models/test_qwen3_next_models_pcg.py b/test/srt/models/test_qwen3_next_models_pcg.py new file mode 100644 index 000000000..e968cbd63 --- /dev/null +++ b/test/srt/models/test_qwen3_next_models_pcg.py @@ -0,0 +1,70 @@ +""" +Qwen3 Next piecewise CUDA graph tests. + +DISABLED: See https://github.com/sgl-project/sglang/issues/17039 +PCG tests for Qwen3 Next have intermittent failures (5-10% probability). +Investigation ongoing by @YuweiAn. +""" + +import unittest +from types import SimpleNamespace + +from sglang.srt.utils import kill_process_tree +from sglang.test.few_shot_gsm8k import run_eval +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + +QWEN3_NEXT_MODEL = "Qwen/Qwen3-Next-80B-A3B-Instruct" + +ACC_THRESHOLDS = { + QWEN3_NEXT_MODEL: {"kl_div": 0.0025, "gsm8k": 0.93}, +} + + +@unittest.skip("Disabled: intermittent failures, see #17039") +class TestQwen3NextPiecewiseCudaGraph(CustomTestCase): + + @classmethod + def setUpClass(cls): + cls.model = QWEN3_NEXT_MODEL + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--tp", + "4", + "--enable-piecewise-cuda-graph", + "--piecewise-cuda-graph-compiler", + "eager", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval(args) + print(f"{metrics=}") + self.assertGreaterEqual( + metrics["accuracy"], ACC_THRESHOLDS[self.model]["gsm8k"] + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/run_suite.py b/test/srt/run_suite.py index 3e1f0237e..fdefc0992 100644 --- a/test/srt/run_suite.py +++ b/test/srt/run_suite.py @@ -9,7 +9,8 @@ from sglang.test.ci.ci_utils import TestFile, run_unittest_files # NOTE: please sort the test cases alphabetically by the test file name suites = { "per-commit-4-gpu": [ - TestFile("models/test_qwen3_next_models.py", 650), + TestFile("models/test_qwen3_next_models.py", 350), + TestFile("models/test_qwen3_next_models_mtp.py", 500), TestFile("test_gpt_oss_4gpu.py", 300), TestFile("test_multi_instance_release_memory_occupation.py", 64), TestFile("test_pp_single_node.py", 500), @@ -70,6 +71,9 @@ suites = { TestFile("test_mistral_large3_basic.py"), TestFile("test_prefill_delayer.py"), TestFile("test_fla_layernorm_guard.py"), + TestFile( + "models/test_qwen3_next_models_pcg.py" + ), # Disabled: intermittent failures, see #17039 ], }