import unittest from types import SimpleNamespace from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k from sglang.test.server_fixtures.disaggregation_fixture import ( PDDisaggregationServerBase, ) from sglang.test.test_utils import ( DEFAULT_MODEL_NAME_FOR_TEST, DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, popen_launch_pd_server, ) class TestDisaggregationPiecewiseCudaGraph(PDDisaggregationServerBase): """Test piecewise CUDA graph support in disaggregation prefill server""" @classmethod def setUpClass(cls): super().setUpClass() cls.model = DEFAULT_MODEL_NAME_FOR_TEST # Start servers cls.start_prefill() cls.start_decode() # Wait for both to be ready cls.wait_server_ready(cls.prefill_url + "/health", process=cls.process_prefill) cls.wait_server_ready(cls.decode_url + "/health", process=cls.process_decode) cls.launch_lb() @classmethod def start_prefill(cls): prefill_args = [ "--trust-remote-code", "--disaggregation-mode", "prefill", "--tp", "1", "--enable-piecewise-cuda-graph", ] prefill_args += cls.transfer_backend + cls.rdma_devices cls.process_prefill = popen_launch_pd_server( cls.model, cls.prefill_url, timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, other_args=prefill_args, ) @classmethod def start_decode(cls): decode_args = [ "--trust-remote-code", "--disaggregation-mode", "decode", "--tp", "1", "--base-gpu-id", "1", ] decode_args += cls.transfer_backend + cls.rdma_devices cls.process_decode = popen_launch_pd_server( cls.model, cls.decode_url, timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, other_args=decode_args, ) def test_gsm8k_accuracy(self): """Verify that piecewise cuda graph works correctly in prefill server""" args = SimpleNamespace( num_shots=5, data_path=None, num_questions=200, max_new_tokens=512, parallel=128, host=f"http://{self.base_host}", port=int(self.lb_port), ) metrics = run_eval_few_shot_gsm8k(args) print(f"GSM8K accuracy with piecewise cuda graph: {metrics['accuracy']:.3f}") self.assertGreater(metrics["accuracy"], 0.62) if __name__ == "__main__": unittest.main()