import unittest from types import SimpleNamespace from sglang.test.ci.ci_register import register_cuda_ci from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k from sglang.test.server_fixtures.disaggregation_fixture import ( PDDisaggregationServerBase, ) from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, is_in_ci, popen_launch_pd_server, ) register_cuda_ci( est_time=400, suite="stage-c-test-8-gpu-h200", disabled="TCP fallback flaky" ) @unittest.skipIf(is_in_ci(), "Temporarily disable the flaky test.") class TestDisaggregationHybridAttentionMamba(PDDisaggregationServerBase): @classmethod def setUpClass(cls): super().setUpClass() cls.model = "Qwen/Qwen3-Next-80B-A3B-Instruct" # Non blocking start servers cls.start_prefill() cls.start_decode() # Block until both cls.wait_server_ready(cls.prefill_url + "/health", process=cls.process_prefill) cls.wait_server_ready(cls.decode_url + "/health", process=cls.process_decode) cls.launch_lb() @classmethod def start_prefill(cls): prefill_args = [ "--trust-remote-code", "--disaggregation-mode", "prefill", "--tp", "4", ] prefill_args += cls.transfer_backend + cls.rdma_devices cls.process_prefill = popen_launch_pd_server( cls.model, cls.prefill_url, timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, other_args=prefill_args, ) @classmethod def start_decode(cls): decode_args = [ "--trust-remote-code", "--disaggregation-mode", "decode", "--tp", "4", "--base-gpu-id", "4", ] decode_args += cls.transfer_backend + cls.rdma_devices cls.process_decode = popen_launch_pd_server( cls.model, cls.decode_url, timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, other_args=decode_args, ) def test_gsm8k(self): args = SimpleNamespace( num_shots=5, data_path=None, num_questions=200, max_new_tokens=512, parallel=128, host=f"http://{self.base_host}", port=int(self.lb_port), ) metrics = run_eval_few_shot_gsm8k(args) print(f"Evaluation metrics: {metrics}") self.assertGreater(metrics["accuracy"], 0.93) class TestDisaggregationHybridAttentionMambaExtraBuffer(PDDisaggregationServerBase): @classmethod def setUpClass(cls): super().setUpClass() cls.model = "Qwen/Qwen3-Next-80B-A3B-Instruct" # Non blocking start servers cls.start_prefill() cls.start_decode() # Block until both cls.wait_server_ready(cls.prefill_url + "/health", process=cls.process_prefill) cls.wait_server_ready(cls.decode_url + "/health", process=cls.process_decode) cls.launch_lb() @classmethod def start_prefill(cls): prefill_args = [ "--trust-remote-code", "--disaggregation-mode", "prefill", "--tp", "4", "--mamba-scheduler-strategy", "extra_buffer", ] prefill_args += cls.transfer_backend + cls.rdma_devices cls.process_prefill = popen_launch_pd_server( cls.model, cls.prefill_url, timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, other_args=prefill_args, ) @classmethod def start_decode(cls): decode_args = [ "--trust-remote-code", "--disaggregation-mode", "decode", "--tp", "4", "--base-gpu-id", "4", "--mamba-scheduler-strategy", "extra_buffer", ] decode_args += cls.transfer_backend + cls.rdma_devices cls.process_decode = popen_launch_pd_server( cls.model, cls.decode_url, timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, other_args=decode_args, ) def test_gsm8k(self): args = SimpleNamespace( num_shots=5, data_path=None, num_questions=200, max_new_tokens=512, parallel=128, host=f"http://{self.base_host}", port=int(self.lb_port), ) metrics = run_eval_few_shot_gsm8k(args) print(f"Evaluation metrics: {metrics}") self.assertGreater(metrics["accuracy"], 0.93) @unittest.skipIf( is_in_ci(), "Temporarily disable the flaky test: tcp fallback is not stable currently.", ) class TestDisaggregationHybridAttentionMambaDPDecode(PDDisaggregationServerBase): """Test with prefill tp=2 and decode tp=2/dp=2 with dp-attention enabled.""" @classmethod def setUpClass(cls): super().setUpClass() cls.model = "Qwen/Qwen3-Next-80B-A3B-Instruct" # Non blocking start servers cls.start_prefill() cls.start_decode() # Block until both cls.wait_server_ready(cls.prefill_url + "/health", process=cls.process_prefill) cls.wait_server_ready(cls.decode_url + "/health", process=cls.process_decode) cls.launch_lb() @classmethod def start_prefill(cls): prefill_args = [ "--trust-remote-code", "--disaggregation-mode", "prefill", "--tp", "2", ] prefill_args += cls.transfer_backend + cls.rdma_devices cls.process_prefill = popen_launch_pd_server( cls.model, cls.prefill_url, timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, other_args=prefill_args, ) @classmethod def start_decode(cls): decode_args = [ "--trust-remote-code", "--disaggregation-mode", "decode", "--tp", "2", "--dp", "2", "--enable-dp-attention", "--enable-dp-lm-head", "--base-gpu-id", "2", ] decode_args += cls.transfer_backend + cls.rdma_devices cls.process_decode = popen_launch_pd_server( cls.model, cls.decode_url, timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, other_args=decode_args, ) def test_gsm8k(self): args = SimpleNamespace( num_shots=5, data_path=None, num_questions=200, max_new_tokens=512, parallel=128, host=f"http://{self.base_host}", port=int(self.lb_port), ) metrics = run_eval_few_shot_gsm8k(args) print(f"Evaluation metrics: {metrics}") self.assertGreater(metrics["accuracy"], 0.93) if __name__ == "__main__": unittest.main()