From 9a3bdf2c951670b41e7b3468cb1f85f3217aa40b Mon Sep 17 00:00:00 2001 From: Alison Shao <54658187+alisonshao@users.noreply.github.com> Date: Sat, 20 Dec 2025 18:55:39 -0800 Subject: [PATCH] [CI] Migrate CUDA Graph tests to test/registered/cuda_graph/ (#15436) --- .github/workflows/pr-test.yml | 46 ++++- scripts/ci/slash_command_handler.py | 3 +- .../test_piecewise_cuda_graph_2_gpu.py | 4 + .../test_piecewise_cuda_graph_large_1_gpu.py} | 165 +++++++----------- .../test_piecewise_cuda_graph_small_1_gpu.py} | 141 ++++++++++----- test/registered/lora/test_lora_tp.py | 2 +- test/run_suite.py | 3 +- test/srt/run_suite.py | 3 - 8 files changed, 208 insertions(+), 159 deletions(-) rename test/{srt => registered/cuda_graph}/test_piecewise_cuda_graph_2_gpu.py (94%) rename test/{srt/test_piecewise_cuda_graph_1_gpu_a.py => registered/cuda_graph/test_piecewise_cuda_graph_large_1_gpu.py} (54%) rename test/{srt/test_piecewise_cuda_graph_1_gpu_b.py => registered/cuda_graph/test_piecewise_cuda_graph_small_1_gpu.py} (72%) diff --git a/.github/workflows/pr-test.yml b/.github/workflows/pr-test.yml index 418260e1e..355668de6 100644 --- a/.github/workflows/pr-test.yml +++ b/.github/workflows/pr-test.yml @@ -448,12 +448,49 @@ jobs: cd test/ python3 run_suite.py --hw cuda --suite stage-b-test-small-1-gpu --auto-partition-id ${{ matrix.partition }} --auto-partition-size 3 - stage-b-test-2-gpu: + stage-b-test-large-1-gpu: needs: [check-changes, call-gate, stage-a-test-1, sgl-kernel-build-wheels] if: | always() && ( - (inputs.target_stage == 'stage-b-test-2-gpu') || + (inputs.target_stage == 'stage-b-test-large-1-gpu') || + ( + !inputs.target_stage && + (github.event_name == 'schedule' || (!failure() && !cancelled())) && + ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true')) + ) + ) + runs-on: 1-gpu-runner + env: + RUNNER_LABELS: 1-gpu-runner + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Download artifacts + if: needs.check-changes.outputs.sgl_kernel == 'true' + uses: actions/download-artifact@v4 + with: + path: sgl-kernel/dist/ + merge-multiple: true + pattern: wheel-python3.10-cuda12.9 + + - name: Install dependencies + run: | + CUSTOM_BUILD_SGL_KERNEL=${{needs.check-changes.outputs.sgl_kernel}} bash scripts/ci/ci_install_dependency.sh + + - name: Run test + timeout-minutes: 30 + run: | + cd test/ + python3 run_suite.py --hw cuda --suite stage-b-test-large-1-gpu + + stage-b-test-large-2-gpu: + needs: [check-changes, call-gate, stage-a-test-1, sgl-kernel-build-wheels] + if: | + always() && + ( + (inputs.target_stage == 'stage-b-test-large-2-gpu') || ( !inputs.target_stage && (github.event_name == 'schedule' || (!failure() && !cancelled())) && @@ -483,7 +520,7 @@ jobs: timeout-minutes: 30 run: | cd test/ - python3 run_suite.py --hw cuda --suite stage-b-test-small-2-gpu + python3 run_suite.py --hw cuda --suite stage-b-test-large-2-gpu multimodal-gen-test-1-gpu: needs: [check-changes, call-gate, sgl-kernel-build-wheels] @@ -1363,7 +1400,8 @@ jobs: stage-a-test-1, stage-b-test-small-1-gpu, - stage-b-test-2-gpu, + stage-b-test-large-1-gpu, + stage-b-test-large-2-gpu, quantization-test, unit-test-backend-1-gpu, unit-test-backend-2-gpu, diff --git a/scripts/ci/slash_command_handler.py b/scripts/ci/slash_command_handler.py index f43505d0f..ad04de6ad 100644 --- a/scripts/ci/slash_command_handler.py +++ b/scripts/ci/slash_command_handler.py @@ -144,7 +144,8 @@ def handle_rerun_stage( nvidia_stages = [ "stage-a-test-1", "stage-b-test-small-1-gpu", - "stage-b-test-2-gpu", + "stage-b-test-large-1-gpu", + "stage-b-test-large-2-gpu", "multimodal-gen-test-1-gpu", "multimodal-gen-test-2-gpu", "quantization-test", diff --git a/test/srt/test_piecewise_cuda_graph_2_gpu.py b/test/registered/cuda_graph/test_piecewise_cuda_graph_2_gpu.py similarity index 94% rename from test/srt/test_piecewise_cuda_graph_2_gpu.py rename to test/registered/cuda_graph/test_piecewise_cuda_graph_2_gpu.py index 100edae65..b99b5d9ef 100644 --- a/test/srt/test_piecewise_cuda_graph_2_gpu.py +++ b/test/registered/cuda_graph/test_piecewise_cuda_graph_2_gpu.py @@ -1,6 +1,7 @@ import unittest from sglang.srt.utils import kill_process_tree +from sglang.test.ci.ci_register import register_cuda_ci from sglang.test.run_eval import run_eval from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, @@ -10,6 +11,9 @@ from sglang.test.test_utils import ( popen_launch_server, ) +# CI Registration - 2-GPU tests (80GB GPUs required) +register_cuda_ci(est_time=255, suite="stage-b-test-large-2-gpu") + class TestPiecewiseCudaGraphQwen3OmniMOE(CustomTestCase): """Test piecewise CUDA graph with Qwen3-Omni-30B-A3B-Instruct model""" diff --git a/test/srt/test_piecewise_cuda_graph_1_gpu_a.py b/test/registered/cuda_graph/test_piecewise_cuda_graph_large_1_gpu.py similarity index 54% rename from test/srt/test_piecewise_cuda_graph_1_gpu_a.py rename to test/registered/cuda_graph/test_piecewise_cuda_graph_large_1_gpu.py index a72759a4e..8814cdcd3 100644 --- a/test/srt/test_piecewise_cuda_graph_1_gpu_a.py +++ b/test/registered/cuda_graph/test_piecewise_cuda_graph_large_1_gpu.py @@ -1,96 +1,18 @@ import unittest -from sglang.srt.utils import get_device_sm, kill_process_tree -from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.srt.utils import kill_process_tree +from sglang.test.ci.ci_register import register_cuda_ci from sglang.test.run_eval import run_eval from sglang.test.test_utils import ( - DEFAULT_MODEL_NAME_FOR_TEST, - DEFAULT_MODEL_NAME_FOR_TEST_MLA, DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, DEFAULT_URL_FOR_TEST, CustomTestCase, SimpleNamespace, popen_launch_server, - run_bench_one_batch, ) - -class TestPiecewiseCudaGraphCorrectness(CustomTestCase): - @classmethod - def setUpClass(cls): - cls.model = DEFAULT_MODEL_NAME_FOR_TEST - cls.base_url = DEFAULT_URL_FOR_TEST - cls.process = popen_launch_server( - cls.model, - cls.base_url, - timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - other_args=["--enable-piecewise-cuda-graph"], - ) - - @classmethod - def tearDownClass(cls): - kill_process_tree(cls.process.pid) - - def test_mmlu(self): - args = SimpleNamespace( - base_url=self.base_url, - model=self.model, - eval_name="mmlu", - num_examples=64, - num_threads=32, - ) - - metrics = run_eval(args) - self.assertGreaterEqual(metrics["score"], 0.65) - - -class TestPiecewiseCudaGraphBenchmark(CustomTestCase): - - def test_latency(self): - prefill_latency, _, _ = run_bench_one_batch( - DEFAULT_MODEL_NAME_FOR_TEST, - other_args=["--enable-piecewise-cuda-graph"], - ) - self.assertLess(prefill_latency, 0.015) - - -@unittest.skipIf(get_device_sm() < 100, "Test requires CUDA SM 100 or higher") -class TestPiecewiseCudaGraphLlama31FP4(CustomTestCase): - """MGSM test: piecewise CUDA graph with NVFP4 Llama3.1 8B on Blackwell.""" - - @classmethod - def setUpClass(cls): - cls.model = "nvidia/Llama-3.1-8B-Instruct-FP4" - cls.base_url = DEFAULT_URL_FOR_TEST - cls.process = popen_launch_server( - cls.model, - cls.base_url, - timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - other_args=[ - "--enable-piecewise-cuda-graph", - "--quantization", - "modelopt_fp4", - "--mem-fraction-static", - "0.8", - ], - ) - - @classmethod - def tearDownClass(cls): - kill_process_tree(cls.process.pid) - - def test_mgsm_accuracy(self): - num_examples = 1319 - args = SimpleNamespace( - base_url=self.base_url, - model=self.model, - eval_name="mgsm_en", - num_examples=num_examples, - num_threads=min(num_examples, 1024), - ) - metrics = run_eval(args) - print(f"MGSM Accuracy: {metrics['score']:.3f}") - self.assertGreaterEqual(metrics["score"], 0.78) +# CI Registration - Large 1-GPU tests (80GB GPU required) +register_cuda_ci(est_time=480, suite="stage-b-test-large-1-gpu") class TestPiecewiseCudaGraphQwen3MoE(CustomTestCase): @@ -133,42 +55,77 @@ class TestPiecewiseCudaGraphQwen3MoE(CustomTestCase): self.assertGreaterEqual(metrics["score"], 0.90) -class TestPiecewiseCudaGraphDeepSeek(CustomTestCase): +class TestPiecewiseCudaGraphGPTQ(CustomTestCase): + @classmethod def setUpClass(cls): - cls.model = DEFAULT_MODEL_NAME_FOR_TEST_MLA + cls.model = "Qwen/Qwen3-30B-A3B-GPTQ-Int4" cls.base_url = DEFAULT_URL_FOR_TEST cls.process = popen_launch_server( cls.model, cls.base_url, timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - other_args=[ - "--enable-piecewise-cuda-graph", - "--piecewise-cuda-graph-compiler", - "eager", - "--piecewise-cuda-graph-max-tokens", - "4096", # should less than max_context_len - ], + other_args=["--enable-piecewise-cuda-graph"], ) @classmethod def tearDownClass(cls): kill_process_tree(cls.process.pid) - def test_gsm8k(self): - args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=200, - max_new_tokens=512, - parallel=128, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), - ) - metrics = run_eval_few_shot_gsm8k(args) - print(metrics) + def test_mgsm_accuracy(self): + num_examples = 1319 - self.assertGreater(metrics["accuracy"], 0.62) + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mgsm_en", + num_examples=num_examples, + num_threads=min(num_examples, 1024), + ) + + metrics = run_eval(args) + print(f"MGSM Accuracy: {metrics['score']:.3f}") + + # Expected accuracy: 0.948, allow some variance + self.assertGreaterEqual(metrics["score"], 0.92) + + +class TestPiecewiseCudaGraphAWQ(CustomTestCase): + """Test piecewise CUDA graph with AWQ quantized model""" + + @classmethod + def setUpClass(cls): + cls.model = "Qwen/QwQ-32B-AWQ" + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=["--enable-piecewise-cuda-graph"], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mgsm_accuracy(self): + """Test MGSM accuracy with AWQ model""" + num_examples = 1319 + + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mgsm_en", + num_examples=num_examples, + num_threads=min(num_examples, 1024), + ) + + metrics = run_eval(args) + print(f"MGSM Accuracy: {metrics['score']:.3f}") + print(f"Output throughput: {metrics.get('throughput', 'N/A')} token/s") + + # Expected accuracy: 0.680, allow some variance + self.assertGreaterEqual(metrics["score"], 0.65) if __name__ == "__main__": diff --git a/test/srt/test_piecewise_cuda_graph_1_gpu_b.py b/test/registered/cuda_graph/test_piecewise_cuda_graph_small_1_gpu.py similarity index 72% rename from test/srt/test_piecewise_cuda_graph_1_gpu_b.py rename to test/registered/cuda_graph/test_piecewise_cuda_graph_small_1_gpu.py index dc2502eb4..bb180bd27 100644 --- a/test/srt/test_piecewise_cuda_graph_1_gpu_b.py +++ b/test/registered/cuda_graph/test_piecewise_cuda_graph_small_1_gpu.py @@ -4,23 +4,30 @@ import torch from sglang import Engine from sglang.lang.chat_template import get_chat_template_by_model_path -from sglang.srt.utils import kill_process_tree +from sglang.srt.utils import get_device_sm, kill_process_tree +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k from sglang.test.run_eval import run_eval from sglang.test.test_utils import ( DEFAULT_IMAGE_URL, + DEFAULT_MODEL_NAME_FOR_TEST, + DEFAULT_MODEL_NAME_FOR_TEST_MLA, DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, DEFAULT_URL_FOR_TEST, CustomTestCase, SimpleNamespace, popen_launch_server, + run_bench_one_batch, ) +# CI Registration - Small 1-GPU tests (24GB GPU sufficient) +register_cuda_ci(est_time=460, suite="stage-b-test-small-1-gpu") -class TestPiecewiseCudaGraphGPTQ(CustomTestCase): +class TestPiecewiseCudaGraphCorrectness(CustomTestCase): @classmethod def setUpClass(cls): - cls.model = "Qwen/Qwen3-30B-A3B-GPTQ-Int4" + cls.model = DEFAULT_MODEL_NAME_FOR_TEST cls.base_url = DEFAULT_URL_FOR_TEST cls.process = popen_launch_server( cls.model, @@ -33,9 +40,56 @@ class TestPiecewiseCudaGraphGPTQ(CustomTestCase): def tearDownClass(cls): kill_process_tree(cls.process.pid) + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + self.assertGreaterEqual(metrics["score"], 0.65) + + +class TestPiecewiseCudaGraphBenchmark(CustomTestCase): + + def test_latency(self): + prefill_latency, _, _ = run_bench_one_batch( + DEFAULT_MODEL_NAME_FOR_TEST, + other_args=["--enable-piecewise-cuda-graph"], + ) + self.assertLess(prefill_latency, 0.015) + + +@unittest.skipIf(get_device_sm() < 100, "Test requires CUDA SM 100 or higher") +class TestPiecewiseCudaGraphLlama31FP4(CustomTestCase): + """MGSM test: piecewise CUDA graph with NVFP4 Llama3.1 8B on Blackwell.""" + + @classmethod + def setUpClass(cls): + cls.model = "nvidia/Llama-3.1-8B-Instruct-FP4" + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--enable-piecewise-cuda-graph", + "--quantization", + "modelopt_fp4", + "--mem-fraction-static", + "0.8", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + def test_mgsm_accuracy(self): num_examples = 1319 - args = SimpleNamespace( base_url=self.base_url, model=self.model, @@ -43,12 +97,47 @@ class TestPiecewiseCudaGraphGPTQ(CustomTestCase): num_examples=num_examples, num_threads=min(num_examples, 1024), ) - metrics = run_eval(args) print(f"MGSM Accuracy: {metrics['score']:.3f}") + self.assertGreaterEqual(metrics["score"], 0.78) - # Expected accuracy: 0.948, allow some variance - self.assertGreaterEqual(metrics["score"], 0.92) + +class TestPiecewiseCudaGraphDeepSeek(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_MODEL_NAME_FOR_TEST_MLA + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--enable-piecewise-cuda-graph", + "--piecewise-cuda-graph-compiler", + "eager", + "--piecewise-cuda-graph-max-tokens", + "4096", # should less than max_context_len + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval_few_shot_gsm8k(args) + print(metrics) + + self.assertGreater(metrics["accuracy"], 0.62) class TestPiecewiseCudaGraphFP8(CustomTestCase): @@ -208,43 +297,5 @@ class TestPiecewiseCudaGraphQwen25VLEmbedding(CustomTestCase): ) -class TestPiecewiseCudaGraphAWQ(CustomTestCase): - """Test piecewise CUDA graph with AWQ quantized model""" - - @classmethod - def setUpClass(cls): - cls.model = "Qwen/QwQ-32B-AWQ" - cls.base_url = DEFAULT_URL_FOR_TEST - cls.process = popen_launch_server( - cls.model, - cls.base_url, - timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - other_args=["--enable-piecewise-cuda-graph"], - ) - - @classmethod - def tearDownClass(cls): - kill_process_tree(cls.process.pid) - - def test_mgsm_accuracy(self): - """Test MGSM accuracy with AWQ model""" - num_examples = 1319 - - args = SimpleNamespace( - base_url=self.base_url, - model=self.model, - eval_name="mgsm_en", - num_examples=num_examples, - num_threads=min(num_examples, 1024), - ) - - metrics = run_eval(args) - print(f"MGSM Accuracy: {metrics['score']:.3f}") - print(f"Output throughput: {metrics.get('throughput', 'N/A')} token/s") - - # Expected accuracy: 0.680, allow some variance - self.assertGreaterEqual(metrics["score"], 0.65) - - if __name__ == "__main__": unittest.main() diff --git a/test/registered/lora/test_lora_tp.py b/test/registered/lora/test_lora_tp.py index c90979c9a..a8cddf6d6 100644 --- a/test/registered/lora/test_lora_tp.py +++ b/test/registered/lora/test_lora_tp.py @@ -28,7 +28,7 @@ from sglang.test.lora_utils import ( ) from sglang.test.test_utils import CustomTestCase, is_in_ci -register_cuda_ci(est_time=116, suite="stage-b-test-small-2-gpu") +register_cuda_ci(est_time=116, suite="stage-b-test-large-2-gpu") class TestLoRATP(CustomTestCase): diff --git a/test/run_suite.py b/test/run_suite.py index 1986079b9..6f9b610d2 100644 --- a/test/run_suite.py +++ b/test/run_suite.py @@ -22,7 +22,8 @@ PER_COMMIT_SUITES = { HWBackend.CUDA: [ "stage-a-test-1", "stage-b-test-small-1-gpu", - "stage-b-test-small-2-gpu", + "stage-b-test-large-1-gpu", + "stage-b-test-large-2-gpu", ], HWBackend.NPU: [], } diff --git a/test/srt/run_suite.py b/test/srt/run_suite.py index 14552ca46..f6de873e4 100644 --- a/test/srt/run_suite.py +++ b/test/srt/run_suite.py @@ -88,8 +88,6 @@ suites = { TestFile("test_original_logprobs.py", 41), TestFile("test_page_size.py", 60), TestFile("test_penalty.py", 82), - TestFile("test_piecewise_cuda_graph_1_gpu_a.py", 460), - TestFile("test_piecewise_cuda_graph_1_gpu_b.py", 480), TestFile("test_priority_scheduling.py", 130), TestFile("test_pytorch_sampling_backend.py", 66), TestFile("test_radix_attention.py", 105), @@ -140,7 +138,6 @@ suites = { TestFile("test_dp_attention.py", 350), TestFile("test_load_weights_from_remote_instance.py", 72), TestFile("test_patch_torch.py", 19), - TestFile("test_piecewise_cuda_graph_2_gpu.py", 400), TestFile("test_eagle_dp_attention.py", 200), ], "per-commit-4-gpu": [