chenchenghao/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[CI] Migrate CUDA Graph tests to test/registered/cuda_graph/ (#15436)

Browse Source
This commit is contained in:
Alison Shao
2025-12-20 18:55:39 -08:00
committed by GitHub
parent 4b351f6b95
commit 9a3bdf2c95
8 changed files with 208 additions and 159 deletions
Download Patch File Download Diff File Expand all files Collapse all files

46
.github/workflows/pr-test.yml vendored
View File

@@ -448,12 +448,49 @@ jobs:
cd test/
python3 run_suite.py --hw cuda --suite stage-b-test-small-1-gpu --auto-partition-id ${{ matrix.partition }} --auto-partition-size 3
stage-b-test-2-gpu:
stage-b-test-large-1-gpu:
needs: [check-changes, call-gate, stage-a-test-1, sgl-kernel-build-wheels]
if: |
always() &&
(
(inputs.target_stage == 'stage-b-test-2-gpu') ||
(inputs.target_stage == 'stage-b-test-large-1-gpu') ||
(
!inputs.target_stage &&
(github.event_name == 'schedule' || (!failure() && !cancelled())) &&
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
)
)
runs-on: 1-gpu-runner
env:
RUNNER_LABELS: 1-gpu-runner
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Download artifacts
if: needs.check-changes.outputs.sgl_kernel == 'true'
uses: actions/download-artifact@v4
with:
path: sgl-kernel/dist/
merge-multiple: true
pattern: wheel-python3.10-cuda12.9
- name: Install dependencies
run: |
CUSTOM_BUILD_SGL_KERNEL=${{needs.check-changes.outputs.sgl_kernel}} bash scripts/ci/ci_install_dependency.sh
- name: Run test
timeout-minutes: 30
run: |
cd test/
python3 run_suite.py --hw cuda --suite stage-b-test-large-1-gpu
stage-b-test-large-2-gpu:
needs: [check-changes, call-gate, stage-a-test-1, sgl-kernel-build-wheels]
if: |
always() &&
(
(inputs.target_stage == 'stage-b-test-large-2-gpu') ||
(
!inputs.target_stage &&
(github.event_name == 'schedule' || (!failure() && !cancelled())) &&
@@ -483,7 +520,7 @@ jobs:
timeout-minutes: 30
run: |
cd test/
python3 run_suite.py --hw cuda --suite stage-b-test-small-2-gpu
python3 run_suite.py --hw cuda --suite stage-b-test-large-2-gpu
multimodal-gen-test-1-gpu:
needs: [check-changes, call-gate, sgl-kernel-build-wheels]
@@ -1363,7 +1400,8 @@ jobs:
stage-a-test-1,
stage-b-test-small-1-gpu,
stage-b-test-2-gpu,
stage-b-test-large-1-gpu,
stage-b-test-large-2-gpu,
quantization-test,
unit-test-backend-1-gpu,
unit-test-backend-2-gpu,

3
scripts/ci/slash_command_handler.py
View File

@@ -144,7 +144,8 @@ def handle_rerun_stage(
nvidia_stages = [
"stage-a-test-1",
"stage-b-test-small-1-gpu",
"stage-b-test-2-gpu",
"stage-b-test-large-1-gpu",
"stage-b-test-large-2-gpu",
"multimodal-gen-test-1-gpu",
"multimodal-gen-test-2-gpu",
"quantization-test",

4
test/srt/test_piecewise_cuda_graph_2_gpu.py → test/registered/cuda_graph/test_piecewise_cuda_graph_2_gpu.py
View File

@@ -1,6 +1,7 @@
import unittest
from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.run_eval import run_eval
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
@@ -10,6 +11,9 @@ from sglang.test.test_utils import (
popen_launch_server,
)
# CI Registration - 2-GPU tests (80GB GPUs required)
register_cuda_ci(est_time=255, suite="stage-b-test-large-2-gpu")
class TestPiecewiseCudaGraphQwen3OmniMOE(CustomTestCase):
"""Test piecewise CUDA graph with Qwen3-Omni-30B-A3B-Instruct model"""

165
test/srt/test_piecewise_cuda_graph_1_gpu_a.py → test/registered/cuda_graph/test_piecewise_cuda_graph_large_1_gpu.py
View File

@@ -1,96 +1,18 @@
import unittest
from sglang.srt.utils import get_device_sm, kill_process_tree
from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k
from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.run_eval import run_eval
from sglang.test.test_utils import (
DEFAULT_MODEL_NAME_FOR_TEST,
DEFAULT_MODEL_NAME_FOR_TEST_MLA,
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
CustomTestCase,
SimpleNamespace,
popen_launch_server,
run_bench_one_batch,
)
class TestPiecewiseCudaGraphCorrectness(CustomTestCase):
@classmethod
def setUpClass(cls):
cls.model = DEFAULT_MODEL_NAME_FOR_TEST
cls.base_url = DEFAULT_URL_FOR_TEST
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=["--enable-piecewise-cuda-graph"],
)
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
def test_mmlu(self):
args = SimpleNamespace(
base_url=self.base_url,
model=self.model,
eval_name="mmlu",
num_examples=64,
num_threads=32,
)
metrics = run_eval(args)
self.assertGreaterEqual(metrics["score"], 0.65)
class TestPiecewiseCudaGraphBenchmark(CustomTestCase):
def test_latency(self):
prefill_latency, _, _ = run_bench_one_batch(
DEFAULT_MODEL_NAME_FOR_TEST,
other_args=["--enable-piecewise-cuda-graph"],
)
self.assertLess(prefill_latency, 0.015)
@unittest.skipIf(get_device_sm() < 100, "Test requires CUDA SM 100 or higher")
class TestPiecewiseCudaGraphLlama31FP4(CustomTestCase):
"""MGSM test: piecewise CUDA graph with NVFP4 Llama3.1 8B on Blackwell."""
@classmethod
def setUpClass(cls):
cls.model = "nvidia/Llama-3.1-8B-Instruct-FP4"
cls.base_url = DEFAULT_URL_FOR_TEST
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=[
"--enable-piecewise-cuda-graph",
"--quantization",
"modelopt_fp4",
"--mem-fraction-static",
"0.8",
],
)
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
def test_mgsm_accuracy(self):
num_examples = 1319
args = SimpleNamespace(
base_url=self.base_url,
model=self.model,
eval_name="mgsm_en",
num_examples=num_examples,
num_threads=min(num_examples, 1024),
)
metrics = run_eval(args)
print(f"MGSM Accuracy: {metrics['score']:.3f}")
self.assertGreaterEqual(metrics["score"], 0.78)
# CI Registration - Large 1-GPU tests (80GB GPU required)
register_cuda_ci(est_time=480, suite="stage-b-test-large-1-gpu")
class TestPiecewiseCudaGraphQwen3MoE(CustomTestCase):
@@ -133,42 +55,77 @@ class TestPiecewiseCudaGraphQwen3MoE(CustomTestCase):
self.assertGreaterEqual(metrics["score"], 0.90)
class TestPiecewiseCudaGraphDeepSeek(CustomTestCase):
class TestPiecewiseCudaGraphGPTQ(CustomTestCase):
@classmethod
def setUpClass(cls):
cls.model = DEFAULT_MODEL_NAME_FOR_TEST_MLA
cls.model = "Qwen/Qwen3-30B-A3B-GPTQ-Int4"
cls.base_url = DEFAULT_URL_FOR_TEST
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=[
"--enable-piecewise-cuda-graph",
"--piecewise-cuda-graph-compiler",
"eager",
"--piecewise-cuda-graph-max-tokens",
"4096", # should less than max_context_len
],
other_args=["--enable-piecewise-cuda-graph"],
)
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
def test_gsm8k(self):
args = SimpleNamespace(
num_shots=5,
data_path=None,
num_questions=200,
max_new_tokens=512,
parallel=128,
host="http://127.0.0.1",
port=int(self.base_url.split(":")[-1]),
)
metrics = run_eval_few_shot_gsm8k(args)
print(metrics)
def test_mgsm_accuracy(self):
num_examples = 1319
self.assertGreater(metrics["accuracy"], 0.62)
args = SimpleNamespace(
base_url=self.base_url,
model=self.model,
eval_name="mgsm_en",
num_examples=num_examples,
num_threads=min(num_examples, 1024),
)
metrics = run_eval(args)
print(f"MGSM Accuracy: {metrics['score']:.3f}")
# Expected accuracy: 0.948, allow some variance
self.assertGreaterEqual(metrics["score"], 0.92)
class TestPiecewiseCudaGraphAWQ(CustomTestCase):
"""Test piecewise CUDA graph with AWQ quantized model"""
@classmethod
def setUpClass(cls):
cls.model = "Qwen/QwQ-32B-AWQ"
cls.base_url = DEFAULT_URL_FOR_TEST
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=["--enable-piecewise-cuda-graph"],
)
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
def test_mgsm_accuracy(self):
"""Test MGSM accuracy with AWQ model"""
num_examples = 1319
args = SimpleNamespace(
base_url=self.base_url,
model=self.model,
eval_name="mgsm_en",
num_examples=num_examples,
num_threads=min(num_examples, 1024),
)
metrics = run_eval(args)
print(f"MGSM Accuracy: {metrics['score']:.3f}")
print(f"Output throughput: {metrics.get('throughput', 'N/A')} token/s")
# Expected accuracy: 0.680, allow some variance
self.assertGreaterEqual(metrics["score"], 0.65)
if __name__ == "__main__":

141
test/srt/test_piecewise_cuda_graph_1_gpu_b.py → test/registered/cuda_graph/test_piecewise_cuda_graph_small_1_gpu.py
View File

@@ -4,23 +4,30 @@ import torch
from sglang import Engine
from sglang.lang.chat_template import get_chat_template_by_model_path
from sglang.srt.utils import kill_process_tree
from sglang.srt.utils import get_device_sm, kill_process_tree
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k
from sglang.test.run_eval import run_eval
from sglang.test.test_utils import (
DEFAULT_IMAGE_URL,
DEFAULT_MODEL_NAME_FOR_TEST,
DEFAULT_MODEL_NAME_FOR_TEST_MLA,
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
CustomTestCase,
SimpleNamespace,
popen_launch_server,
run_bench_one_batch,
)
# CI Registration - Small 1-GPU tests (24GB GPU sufficient)
register_cuda_ci(est_time=460, suite="stage-b-test-small-1-gpu")
class TestPiecewiseCudaGraphGPTQ(CustomTestCase):
class TestPiecewiseCudaGraphCorrectness(CustomTestCase):
@classmethod
def setUpClass(cls):
cls.model = "Qwen/Qwen3-30B-A3B-GPTQ-Int4"
cls.model = DEFAULT_MODEL_NAME_FOR_TEST
cls.base_url = DEFAULT_URL_FOR_TEST
cls.process = popen_launch_server(
cls.model,
@@ -33,9 +40,56 @@ class TestPiecewiseCudaGraphGPTQ(CustomTestCase):
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
def test_mmlu(self):
args = SimpleNamespace(
base_url=self.base_url,
model=self.model,
eval_name="mmlu",
num_examples=64,
num_threads=32,
)
metrics = run_eval(args)
self.assertGreaterEqual(metrics["score"], 0.65)
class TestPiecewiseCudaGraphBenchmark(CustomTestCase):
def test_latency(self):
prefill_latency, _, _ = run_bench_one_batch(
DEFAULT_MODEL_NAME_FOR_TEST,
other_args=["--enable-piecewise-cuda-graph"],
)
self.assertLess(prefill_latency, 0.015)
@unittest.skipIf(get_device_sm() < 100, "Test requires CUDA SM 100 or higher")
class TestPiecewiseCudaGraphLlama31FP4(CustomTestCase):
"""MGSM test: piecewise CUDA graph with NVFP4 Llama3.1 8B on Blackwell."""
@classmethod
def setUpClass(cls):
cls.model = "nvidia/Llama-3.1-8B-Instruct-FP4"
cls.base_url = DEFAULT_URL_FOR_TEST
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=[
"--enable-piecewise-cuda-graph",
"--quantization",
"modelopt_fp4",
"--mem-fraction-static",
"0.8",
],
)
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
def test_mgsm_accuracy(self):
num_examples = 1319
args = SimpleNamespace(
base_url=self.base_url,
model=self.model,
@@ -43,12 +97,47 @@ class TestPiecewiseCudaGraphGPTQ(CustomTestCase):
num_examples=num_examples,
num_threads=min(num_examples, 1024),
)
metrics = run_eval(args)
print(f"MGSM Accuracy: {metrics['score']:.3f}")
self.assertGreaterEqual(metrics["score"], 0.78)
# Expected accuracy: 0.948, allow some variance
self.assertGreaterEqual(metrics["score"], 0.92)
class TestPiecewiseCudaGraphDeepSeek(CustomTestCase):
@classmethod
def setUpClass(cls):
cls.model = DEFAULT_MODEL_NAME_FOR_TEST_MLA
cls.base_url = DEFAULT_URL_FOR_TEST
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=[
"--enable-piecewise-cuda-graph",
"--piecewise-cuda-graph-compiler",
"eager",
"--piecewise-cuda-graph-max-tokens",
"4096", # should less than max_context_len
],
)
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
def test_gsm8k(self):
args = SimpleNamespace(
num_shots=5,
data_path=None,
num_questions=200,
max_new_tokens=512,
parallel=128,
host="http://127.0.0.1",
port=int(self.base_url.split(":")[-1]),
)
metrics = run_eval_few_shot_gsm8k(args)
print(metrics)
self.assertGreater(metrics["accuracy"], 0.62)
class TestPiecewiseCudaGraphFP8(CustomTestCase):
@@ -208,43 +297,5 @@ class TestPiecewiseCudaGraphQwen25VLEmbedding(CustomTestCase):
)
class TestPiecewiseCudaGraphAWQ(CustomTestCase):
"""Test piecewise CUDA graph with AWQ quantized model"""
@classmethod
def setUpClass(cls):
cls.model = "Qwen/QwQ-32B-AWQ"
cls.base_url = DEFAULT_URL_FOR_TEST
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=["--enable-piecewise-cuda-graph"],
)
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
def test_mgsm_accuracy(self):
"""Test MGSM accuracy with AWQ model"""
num_examples = 1319
args = SimpleNamespace(
base_url=self.base_url,
model=self.model,
eval_name="mgsm_en",
num_examples=num_examples,
num_threads=min(num_examples, 1024),
)
metrics = run_eval(args)
print(f"MGSM Accuracy: {metrics['score']:.3f}")
print(f"Output throughput: {metrics.get('throughput', 'N/A')} token/s")
# Expected accuracy: 0.680, allow some variance
self.assertGreaterEqual(metrics["score"], 0.65)
if __name__ == "__main__":
unittest.main()

2
test/registered/lora/test_lora_tp.py
View File

@@ -28,7 +28,7 @@ from sglang.test.lora_utils import (
)
from sglang.test.test_utils import CustomTestCase, is_in_ci
register_cuda_ci(est_time=116, suite="stage-b-test-small-2-gpu")
register_cuda_ci(est_time=116, suite="stage-b-test-large-2-gpu")
class TestLoRATP(CustomTestCase):

3
test/run_suite.py
View File

@@ -22,7 +22,8 @@ PER_COMMIT_SUITES = {
HWBackend.CUDA: [
"stage-a-test-1",
"stage-b-test-small-1-gpu",
"stage-b-test-small-2-gpu",
"stage-b-test-large-1-gpu",
"stage-b-test-large-2-gpu",
],
HWBackend.NPU: [],
}

3
test/srt/run_suite.py
View File

@@ -88,8 +88,6 @@ suites = {
TestFile("test_original_logprobs.py", 41),
TestFile("test_page_size.py", 60),
TestFile("test_penalty.py", 82),
TestFile("test_piecewise_cuda_graph_1_gpu_a.py", 460),
TestFile("test_piecewise_cuda_graph_1_gpu_b.py", 480),
TestFile("test_priority_scheduling.py", 130),
TestFile("test_pytorch_sampling_backend.py", 66),
TestFile("test_radix_attention.py", 105),
@@ -140,7 +138,6 @@ suites = {
TestFile("test_dp_attention.py", 350),
TestFile("test_load_weights_from_remote_instance.py", 72),
TestFile("test_patch_torch.py", 19),
TestFile("test_piecewise_cuda_graph_2_gpu.py", 400),
TestFile("test_eagle_dp_attention.py", 200),
],
"per-commit-4-gpu": [