chenchenghao/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 11 Packages Projects Releases Wiki Activity

Add Deepseek models into nightly tests (#12865)

Browse Source
This commit is contained in:
Kangyan-Zhou
2025-11-08 21:17:49 -08:00
committed by GitHub
parent f290e8016e
commit d134096319
18 changed files with 854 additions and 473 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
test/README.md
View File

@@ -27,6 +27,7 @@ python3 test_choices.py
## Adding or Updating Tests in CI
- Create new test files under `test/srt` or `test/lang` depending on the type of test.
- For nightly tests, place them in `test/srt/nightly/`. Use the `NightlyBenchmarkRunner` helper class in `nightly_utils.py` for performance benchmarking tests.
- Ensure they are referenced in the respective `run_suite.py` (e.g., `test/srt/run_suite.py`) so they are picked up in CI. For most small test cases, they can be added to the `per-commit-1-gpu` suite. Sort the test cases alphabetically by name.
- Ensure you added `unittest.main()` for unittest and `pytest.main([__file__])` for pytest in the scripts. The CI run them via `python3 test_file.py`.
- The CI will run some suites such as `per-commit-1-gpu`, `per-commit-2-gpu`, and `nightly-1-gpu` automatically. If you need special setup or custom test groups, you may modify the workflows in [`.github/workflows/`](https://github.com/sgl-project/sglang/tree/main/.github/workflows).
@@ -46,4 +47,4 @@ python3 test_choices.py
## Adding New Models to Nightly CI
- **For text models**: extend [global model lists variables](https://github.com/sgl-project/sglang/blob/85c1f7937781199203b38bb46325a2840f353a04/python/sglang/test/test_utils.py#L104) in `test_utils.py`, or add more model lists
- **For vlms**: extend the `MODEL_THRESHOLDS` global dictionary in `test_nightly_vlms_.*.py`, see [here](https://github.com/sgl-project/sglang/blob/85c1f7937781199203b38bb46325a2840f353a04/test/srt/test_nightly_vlms_mmmu_eval.py#L19)
- **For vlms**: extend the `MODEL_THRESHOLDS` global dictionary in `test/srt/nightly/test_vlms_mmmu_eval.py`

291
test/srt/nightly/nightly_utils.py Normal file
View File

@@ -0,0 +1,291 @@
"""Utilities for running nightly performance benchmarks with profiling."""
import json
import os
import subprocess
import time
from typing import List, Optional, Tuple
from sglang.bench_one_batch_server import BenchmarkResult, generate_markdown_report
from sglang.srt.utils import kill_process_tree
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
is_in_ci,
popen_launch_server,
write_github_step_summary,
)
class NightlyBenchmarkRunner:
"""Helper class for running nightly performance benchmarks with profiling.
This class encapsulates common patterns used across nightly performance tests,
including profile directory management, benchmark command construction,
result parsing, and report generation.
"""
def __init__(
self,
profile_dir: str,
test_name: str,
base_url: str,
):
"""Initialize the benchmark runner.
Args:
profile_dir: Directory to store performance profiles
test_name: Name of the test (used for reporting)
base_url: Base URL for the server
"""
self.profile_dir = profile_dir
self.test_name = test_name
self.base_url = base_url
self.full_report = f"## {test_name}\n" + BenchmarkResult.help_str()
def setup_profile_directory(self) -> None:
"""Create the profile directory if it doesn't exist."""
os.makedirs(self.profile_dir, exist_ok=True)
def generate_profile_filename(
self, model_path: str, variant: str = ""
) -> Tuple[str, str]:
"""Generate unique profile filename and path for the model.
Args:
model_path: Path to the model (e.g., "deepseek-ai/DeepSeek-V3.1")
variant: Optional variant suffix (e.g., "basic", "mtp", "nsa")
Returns:
Tuple of (profile_path_prefix, json_output_file)
"""
timestamp = int(time.time())
model_safe_name = model_path.replace("/", "_")
# Build filename with optional variant
if variant:
profile_filename = f"{model_safe_name}_{variant}_{timestamp}"
json_filename = f"results_{model_safe_name}_{variant}_{timestamp}.json"
else:
profile_filename = f"{model_safe_name}_{timestamp}"
json_filename = f"results_{model_safe_name}_{timestamp}.json"
profile_path_prefix = os.path.join(self.profile_dir, profile_filename)
return profile_path_prefix, json_filename
def build_benchmark_command(
self,
model_path: str,
batch_sizes: List[int],
input_lens: Tuple[int, ...],
output_lens: Tuple[int, ...],
profile_path_prefix: str,
json_output_file: str,
extra_args: Optional[List[str]] = None,
) -> List[str]:
"""Build the benchmark command with all required arguments.
Args:
model_path: Path to the model
batch_sizes: List of batch sizes to test
input_lens: Tuple of input lengths to test
output_lens: Tuple of output lengths to test
profile_path_prefix: Prefix for profile output files
json_output_file: Path to JSON output file
extra_args: Optional extra arguments to append to command
Returns:
List of command arguments ready for subprocess.run()
"""
command = [
"python3",
"-m",
"sglang.bench_one_batch_server",
"--model",
model_path,
"--base-url",
self.base_url,
"--batch-size",
*[str(x) for x in batch_sizes],
"--input-len",
*[str(x) for x in input_lens],
"--output-len",
*[str(x) for x in output_lens],
"--show-report",
"--profile",
"--profile-by-stage",
"--profile-filename-prefix",
profile_path_prefix,
f"--output-path={json_output_file}",
"--no-append-to-github-summary",
]
if extra_args:
command.extend(extra_args)
return command
def run_benchmark_command(
self, command: List[str], model_description: str = ""
) -> Tuple[subprocess.CompletedProcess, bool]:
"""Execute the benchmark command and return the result.
Args:
command: Command to execute
model_description: Description for logging (e.g., "model_name (variant)")
Returns:
Tuple of (CompletedProcess, success_bool)
"""
print(f"Running command: {' '.join(command)}")
result = subprocess.run(command, capture_output=True, text=True)
if result.returncode != 0:
desc = model_description or "benchmark"
print(f"Error running benchmark for {desc}:")
print(result.stderr)
return result, False
return result, True
def load_benchmark_results(
self, json_output_file: str, model_description: str = ""
) -> Tuple[List[BenchmarkResult], bool]:
"""Load and parse benchmark results from JSON file.
Args:
json_output_file: Path to JSON output file
model_description: Description for logging
Returns:
Tuple of (list of BenchmarkResult objects, success_bool)
"""
benchmark_results = []
if not os.path.exists(json_output_file):
desc = model_description or "model"
print(f"Warning: JSON output file {json_output_file} not found for {desc}")
return benchmark_results, False
try:
with open(json_output_file, "r") as f:
json_data = json.load(f)
# Convert JSON data to BenchmarkResult objects
for data in json_data:
benchmark_result = BenchmarkResult(**data)
benchmark_results.append(benchmark_result)
print(
f"Loaded {len(benchmark_results)} benchmark results from {json_output_file}"
)
# Clean up JSON file
os.remove(json_output_file)
return benchmark_results, True
except Exception as e:
desc = model_description or "model"
print(f"Error loading benchmark results for {desc}: {e}")
# Try to clean up the file anyway
if os.path.exists(json_output_file):
os.remove(json_output_file)
return benchmark_results, False
def run_benchmark_for_model(
self,
model_path: str,
batch_sizes: List[int],
input_lens: Tuple[int, ...],
output_lens: Tuple[int, ...],
other_args: Optional[List[str]] = None,
variant: str = "",
extra_bench_args: Optional[List[str]] = None,
) -> Tuple[List[BenchmarkResult], bool]:
"""Run a complete benchmark for a single model with server management.
This method handles:
- Server launch and cleanup
- Profile filename generation
- Benchmark command construction and execution
- Result loading and parsing
Args:
model_path: Path to the model
batch_sizes: List of batch sizes to test
input_lens: Tuple of input lengths
output_lens: Tuple of output lengths
other_args: Arguments to pass to server launch
variant: Optional variant suffix (e.g., "basic", "mtp")
extra_bench_args: Extra arguments for the benchmark command
Returns:
Tuple of (list of BenchmarkResult objects, success_bool)
"""
benchmark_results = []
model_description = f"{model_path}" + (f" ({variant})" if variant else "")
# Launch server
process = popen_launch_server(
model=model_path,
base_url=self.base_url,
other_args=other_args or [],
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
)
try:
# Generate filenames
profile_path_prefix, json_output_file = self.generate_profile_filename(
model_path, variant
)
# Build and run benchmark command
command = self.build_benchmark_command(
model_path,
batch_sizes,
input_lens,
output_lens,
profile_path_prefix,
json_output_file,
extra_args=extra_bench_args,
)
result, cmd_success = self.run_benchmark_command(command, model_description)
if not cmd_success:
return benchmark_results, False
# Load results
benchmark_results, load_success = self.load_benchmark_results(
json_output_file, model_description
)
return benchmark_results, load_success
finally:
# Always clean up server process
kill_process_tree(process.pid)
def add_report(self, results: List[BenchmarkResult]) -> None:
"""Add benchmark results to the full report.
Args:
results: List of BenchmarkResult objects to add to report
"""
if results:
report_part = generate_markdown_report(self.profile_dir, results)
self.full_report += report_part + "\n"
def write_final_report(self) -> None:
"""Write the final report to GitHub summary if in CI."""
if is_in_ci():
write_github_step_summary(self.full_report)
def get_full_report(self) -> str:
"""Get the accumulated full report.
Returns:
The full markdown report as a string
"""
return self.full_report

98
test/srt/nightly/test_deepseek_v31_perf.py Normal file
View File

@@ -0,0 +1,98 @@
import unittest
from nightly_utils import NightlyBenchmarkRunner
from sglang.test.test_utils import DEFAULT_URL_FOR_TEST, _parse_int_list_env
DEEPSEEK_V31_MODEL_PATH = "deepseek-ai/DeepSeek-V3.1"
PROFILE_DIR = "performance_profiles_deepseek_v31"
class TestNightlyDeepseekV31Basic(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.model = DEEPSEEK_V31_MODEL_PATH
cls.base_url = DEFAULT_URL_FOR_TEST
cls.batch_sizes = [1, 1, 8, 16, 64]
cls.input_lens = tuple(_parse_int_list_env("NIGHTLY_INPUT_LENS", "4096"))
cls.output_lens = tuple(_parse_int_list_env("NIGHTLY_OUTPUT_LENS", "512"))
cls.other_args = [
"--trust-remote-code",
"--tp",
"8",
"--dp",
"8",
"--enable-dp-attention",
]
cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url)
cls.runner.setup_profile_directory()
def test_bench_one_batch(self):
results, success = self.runner.run_benchmark_for_model(
model_path=self.model,
batch_sizes=self.batch_sizes,
input_lens=self.input_lens,
output_lens=self.output_lens,
other_args=self.other_args,
variant="basic",
)
self.runner.add_report(results)
self.runner.write_final_report()
if not success:
raise AssertionError(
f"Benchmark failed for {self.model} with basic configuration"
)
class TestNightlyDeepseekV31MTP(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.model = DEEPSEEK_V31_MODEL_PATH
cls.base_url = DEFAULT_URL_FOR_TEST
cls.batch_sizes = [1, 1, 8, 16, 64]
cls.input_lens = tuple(_parse_int_list_env("NIGHTLY_INPUT_LENS", "4096"))
cls.output_lens = tuple(_parse_int_list_env("NIGHTLY_OUTPUT_LENS", "512"))
cls.other_args = [
"--trust-remote-code",
"--tp",
"8",
"--dp",
"8",
"--enable-dp-attention",
"--speculative-algorithm",
"EAGLE",
"--speculative-num-steps",
"3",
"--speculative-eagle-topk",
"1",
"--speculative-num-draft-tokens",
"4",
"--mem-frac",
"0.7",
]
cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url)
cls.runner.setup_profile_directory()
def test_bench_one_batch(self):
results, success = self.runner.run_benchmark_for_model(
model_path=self.model,
batch_sizes=self.batch_sizes,
input_lens=self.input_lens,
output_lens=self.output_lens,
other_args=self.other_args,
variant="mtp",
)
self.runner.add_report(results)
self.runner.write_final_report()
if not success:
raise AssertionError(
f"Benchmark failed for {self.model} with MTP configuration"
)
if __name__ == "__main__":
unittest.main()

142
test/srt/nightly/test_deepseek_v32_perf.py Normal file
View File

@@ -0,0 +1,142 @@
import unittest
from nightly_utils import NightlyBenchmarkRunner
from sglang.test.test_utils import DEFAULT_URL_FOR_TEST, _parse_int_list_env
DEEPSEEK_V32_MODEL_PATH = "deepseek-ai/DeepSeek-V3.2-Exp"
PROFILE_DIR = "performance_profiles_deepseek_v32"
class TestNightlyDeepseekV32Basic(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.model = DEEPSEEK_V32_MODEL_PATH
cls.base_url = DEFAULT_URL_FOR_TEST
cls.batch_sizes = [1, 1, 8, 16, 64]
cls.input_lens = tuple(_parse_int_list_env("NIGHTLY_INPUT_LENS", "4096"))
cls.output_lens = tuple(_parse_int_list_env("NIGHTLY_OUTPUT_LENS", "512"))
cls.other_args = [
"--trust-remote-code",
"--tp",
"8",
"--dp",
"8",
"--enable-dp-attention",
]
cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url)
cls.runner.setup_profile_directory()
def test_bench_one_batch(self):
results, success = self.runner.run_benchmark_for_model(
model_path=self.model,
batch_sizes=self.batch_sizes,
input_lens=self.input_lens,
output_lens=self.output_lens,
other_args=self.other_args,
variant="basic",
)
self.runner.add_report(results)
self.runner.write_final_report()
if not success:
raise AssertionError(
f"Benchmark failed for {self.model} with basic configuration"
)
class TestNightlyDeepseekV32MTP(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.model = DEEPSEEK_V32_MODEL_PATH
cls.base_url = DEFAULT_URL_FOR_TEST
cls.batch_sizes = [1, 1, 8, 16, 64]
cls.input_lens = tuple(_parse_int_list_env("NIGHTLY_INPUT_LENS", "4096"))
cls.output_lens = tuple(_parse_int_list_env("NIGHTLY_OUTPUT_LENS", "512"))
cls.other_args = [
"--trust-remote-code",
"--tp",
"8",
"--dp",
"8",
"--enable-dp-attention",
"--speculative-algorithm",
"EAGLE",
"--speculative-num-steps",
"3",
"--speculative-eagle-topk",
"1",
"--speculative-num-draft-tokens",
"4",
"--mem-frac",
"0.7",
]
cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url)
cls.runner.setup_profile_directory()
def test_bench_one_batch(self):
results, success = self.runner.run_benchmark_for_model(
model_path=self.model,
batch_sizes=self.batch_sizes,
input_lens=self.input_lens,
output_lens=self.output_lens,
other_args=self.other_args,
variant="mtp",
)
self.runner.add_report(results)
self.runner.write_final_report()
if not success:
raise AssertionError(
f"Benchmark failed for {self.model} with MTP configuration"
)
class TestNightlyDeepseekV32NSA(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.model = DEEPSEEK_V32_MODEL_PATH
cls.base_url = DEFAULT_URL_FOR_TEST
cls.batch_sizes = [1, 1, 8, 16, 64]
cls.input_lens = tuple(_parse_int_list_env("NIGHTLY_INPUT_LENS", "4096"))
cls.output_lens = tuple(_parse_int_list_env("NIGHTLY_OUTPUT_LENS", "512"))
cls.other_args = [
"--trust-remote-code",
"--tp",
"8",
"--dp",
"8",
"--enable-dp-attention",
"--attention-backend",
"nsa",
"--nsa-prefill-backend",
"flashmla_sparse",
"--nsa-decode-backend",
"flashmla_kv",
]
cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url)
cls.runner.setup_profile_directory()
def test_bench_one_batch(self):
results, success = self.runner.run_benchmark_for_model(
model_path=self.model,
batch_sizes=self.batch_sizes,
input_lens=self.input_lens,
output_lens=self.output_lens,
other_args=self.other_args,
variant="nsa",
)
self.runner.add_report(results)
self.runner.write_final_report()
if not success:
raise AssertionError(
f"Benchmark failed for {self.model} with NSA configuration"
)
if __name__ == "__main__":
unittest.main()

58
test/srt/nightly/test_gpt_oss_4gpu_perf.py Normal file
View File

@@ -0,0 +1,58 @@
import unittest
from nightly_utils import NightlyBenchmarkRunner
from sglang.test.test_utils import DEFAULT_URL_FOR_TEST
PROFILE_DIR = "performance_profiles_gpt_oss_4gpu"
class TestNightlyGptOss4GpuPerformance(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.models = [
(
"openai/gpt-oss-120b",
[
"--tp",
"4",
"--cuda-graph-max-bs",
"200",
"--mem-fraction-static",
"0.93",
],
),
]
cls.base_url = DEFAULT_URL_FOR_TEST
cls.batch_sizes = [1, 1, 8, 16, 64]
cls.input_lens = (4096,)
cls.output_lens = (512,)
cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url)
cls.runner.setup_profile_directory()
def test_bench_one_batch(self):
all_model_succeed = True
for model_path, other_args in self.models:
with self.subTest(model=model_path):
results, success = self.runner.run_benchmark_for_model(
model_path=model_path,
batch_sizes=self.batch_sizes,
input_lens=self.input_lens,
output_lens=self.output_lens,
other_args=other_args,
)
if not success:
all_model_succeed = False
self.runner.add_report(results)
self.runner.write_final_report()
if not all_model_succeed:
raise AssertionError("Some models failed the perf tests.")
if __name__ == "__main__":
unittest.main()

0
test/srt/test_nightly_gsm8k_eval_amd.py → test/srt/nightly/test_gsm8k_eval_amd.py
View File

0
test/srt/test_nightly_text_models_gsm8k_eval.py → test/srt/nightly/test_text_models_gsm8k_eval.py
View File

60
test/srt/nightly/test_text_models_perf.py Normal file
View File

@@ -0,0 +1,60 @@
import unittest
from nightly_utils import NightlyBenchmarkRunner
from sglang.test.test_utils import (
DEFAULT_URL_FOR_TEST,
ModelLaunchSettings,
_parse_int_list_env,
parse_models,
)
PROFILE_DIR = "performance_profiles_text_models"
class TestNightlyTextModelsPerformance(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.models = []
# TODO: replace with DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP1 or other model lists
for model_path in parse_models("meta-llama/Llama-3.1-8B-Instruct"):
cls.models.append(ModelLaunchSettings(model_path, tp_size=1))
for model_path in parse_models("Qwen/Qwen2-57B-A14B-Instruct"):
cls.models.append(ModelLaunchSettings(model_path, tp_size=2))
# (parse_models(DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP1), False, False),
# (parse_models(DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP2), False, True),
# (parse_models(DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP1), True, False),
# (parse_models(DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP2), True, True),
cls.base_url = DEFAULT_URL_FOR_TEST
cls.batch_sizes = [1, 1, 8, 16, 64]
cls.input_lens = tuple(_parse_int_list_env("NIGHTLY_INPUT_LENS", "4096"))
cls.output_lens = tuple(_parse_int_list_env("NIGHTLY_OUTPUT_LENS", "512"))
cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url)
cls.runner.setup_profile_directory()
def test_bench_one_batch(self):
all_model_succeed = True
for model_setup in self.models:
with self.subTest(model=model_setup.model_path):
results, success = self.runner.run_benchmark_for_model(
model_path=model_setup.model_path,
batch_sizes=self.batch_sizes,
input_lens=self.input_lens,
output_lens=self.output_lens,
other_args=model_setup.extra_args,
)
if not success:
all_model_succeed = False
self.runner.add_report(results)
self.runner.write_final_report()
if not all_model_succeed:
raise AssertionError("Some models failed the perf tests.")
if __name__ == "__main__":
unittest.main()

0
test/srt/test_nightly_vlms_mmmu_eval.py → test/srt/nightly/test_vlms_mmmu_eval.py
View File

88
test/srt/nightly/test_vlms_perf.py Normal file
View File

@@ -0,0 +1,88 @@
import os
import unittest
import warnings
from nightly_utils import NightlyBenchmarkRunner
from sglang.test.test_utils import (
DEFAULT_URL_FOR_TEST,
ModelLaunchSettings,
_parse_int_list_env,
parse_models,
)
PROFILE_DIR = "performance_profiles_vlms"
MODEL_DEFAULTS = [
# Keep conservative defaults. Can be overridden by env NIGHTLY_VLM_MODELS
ModelLaunchSettings(
"Qwen/Qwen2.5-VL-7B-Instruct",
extra_args=["--mem-fraction-static=0.7"],
),
ModelLaunchSettings(
"google/gemma-3-27b-it",
),
ModelLaunchSettings("Qwen/Qwen3-VL-30B-A3B-Instruct", extra_args=["--tp=2"]),
# "OpenGVLab/InternVL2_5-2B",
# buggy in official transformers impl
# "openbmb/MiniCPM-V-2_6",
]
class TestNightlyVLMModelsPerformance(unittest.TestCase):
@classmethod
def setUpClass(cls):
warnings.filterwarnings(
"ignore", category=ResourceWarning, message="unclosed.*socket"
)
nightly_vlm_models_str = os.environ.get("NIGHTLY_VLM_MODELS")
if nightly_vlm_models_str:
cls.models = []
model_paths = parse_models(nightly_vlm_models_str)
for model_path in model_paths:
cls.models.append(ModelLaunchSettings(model_path))
else:
cls.models = MODEL_DEFAULTS
cls.base_url = DEFAULT_URL_FOR_TEST
cls.batch_sizes = _parse_int_list_env("NIGHTLY_VLM_BATCH_SIZES", "1,1,2,8,16")
cls.input_lens = tuple(_parse_int_list_env("NIGHTLY_VLM_INPUT_LENS", "4096"))
cls.output_lens = tuple(_parse_int_list_env("NIGHTLY_VLM_OUTPUT_LENS", "512"))
cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url)
cls.runner.setup_profile_directory()
def test_bench_one_batch(self):
all_model_succeed = True
for model_setup in self.models:
with self.subTest(model=model_setup.model_path):
# VLMs need additional benchmark args for dataset and trust-remote-code
extra_bench_args = [
"--trust-remote-code",
"--dataset-name=mmmu",
]
results, success = self.runner.run_benchmark_for_model(
model_path=model_setup.model_path,
batch_sizes=self.batch_sizes,
input_lens=self.input_lens,
output_lens=self.output_lens,
other_args=model_setup.extra_args,
extra_bench_args=extra_bench_args,
)
if not success:
all_model_succeed = False
self.runner.add_report(results)
self.runner.write_final_report()
if not all_model_succeed:
raise AssertionError("Some models failed the perf tests.")
if __name__ == "__main__":
unittest.main()

18
test/srt/run_suite.py
View File

@@ -219,8 +219,9 @@ suites = {
],
"nightly-4-gpu-b200": [
TestFile("test_fp4_moe.py", 300),
TestFile("test_nightly_gpt_oss_4gpu_perf.py", 600),
TestFile("nightly/test_nightly_gpt_oss_4gpu_perf.py", 600),
],
"nightly-8-gpu-b200": [],
"nightly-4-gpu": [],
"nightly-8-gpu": [],
"nightly-8-gpu-h200": [],
@@ -334,11 +335,14 @@ suites = {
TestFile("test_moe_ep.py"),
TestFile("test_moe_eval_accuracy_large.py"),
TestFile("test_mscclpp.py"),
TestFile("test_nightly_gsm8k_eval.py"),
TestFile("test_nightly_text_models_gsm8k_eval.py"),
TestFile("test_nightly_text_models_perf.py"),
TestFile("test_nightly_vlms_mmmu_eval.py"),
TestFile("test_nightly_vlms_perf.py"),
TestFile("nightly/test_deepseek_v31_perf.py"),
TestFile("nightly/test_deepseek_v32_perf.py"),
TestFile("nightly/test_gpt_oss_4gpu_perf.py"),
TestFile("nightly/test_gsm8k_eval_amd.py"),
TestFile("nightly/test_text_models_gsm8k_eval.py"),
TestFile("nightly/test_text_models_perf.py"),
TestFile("nightly/test_vlms_mmmu_eval.py"),
TestFile("nightly/test_vlms_perf.py"),
TestFile("test_openai_adapter.py"),
TestFile("test_openai_function_calling.py"),
TestFile("test_openai_server.py"),
@@ -478,7 +482,7 @@ suite_amd = {
TestFile("test_deepseek_v3_mtp.py", 275),
],
"nightly-amd": [
TestFile("test_nightly_gsm8k_eval_amd.py"),
TestFile("nightly/test_gsm8k_eval_amd.py"),
],
}

134
test/srt/test_nightly_gpt_oss_4gpu_perf.py
View File

@@ -1,134 +0,0 @@
import os
import subprocess
import time
import unittest
from sglang.bench_one_batch_server import BenchmarkResult, generate_markdown_report
from sglang.srt.utils import kill_process_tree
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
is_in_ci,
popen_launch_server,
write_github_step_summary,
)
PROFILE_DIR = "performance_profiles_gpt_oss_4gpu"
class TestNightlyGptOss4GpuPerformance(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.models = [
("lmsys/gpt-oss-120b-bf16", ["--tp", "4", "--cuda-graph-max-bs", "200"]),
(
"openai/gpt-oss-120b",
[
"--tp",
"4",
"--cuda-graph-max-bs",
"200",
"--mem-fraction-static",
"0.93",
],
),
]
cls.base_url = DEFAULT_URL_FOR_TEST
cls.batch_sizes = [1, 1, 8, 16, 64]
cls.input_lens = (4096,)
cls.output_lens = (512,)
os.makedirs(PROFILE_DIR, exist_ok=True)
cls.full_report = f"## {cls.__name__}\n" + BenchmarkResult.help_str()
def test_bench_one_batch(self):
all_benchmark_results = []
all_model_succeed = True
for model_path, other_args in self.models:
benchmark_results = []
with self.subTest(model=model_path):
process = popen_launch_server(
model=model_path,
base_url=self.base_url,
other_args=other_args,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
)
try:
profile_filename = (
f"{model_path.replace('/', '_')}_{int(time.time())}"
)
profile_path_prefix = os.path.join(PROFILE_DIR, profile_filename)
json_output_file = f"results_{model_path.replace('/', '_')}_{int(time.time())}.json"
command = [
"python3",
"-m",
"sglang.bench_one_batch_server",
"--model",
model_path,
"--base-url",
self.base_url,
"--batch-size",
*[str(x) for x in self.batch_sizes],
"--input-len",
*[str(x) for x in self.input_lens],
"--output-len",
*[str(x) for x in self.output_lens],
"--show-report",
"--profile",
"--profile-by-stage",
"--profile-filename-prefix",
profile_path_prefix,
f"--output-path={json_output_file}",
"--no-append-to-github-summary",
]
print(f"Running command: {' '.join(command)}")
result = subprocess.run(command, capture_output=True, text=True)
if result.returncode != 0:
print(
f"Error running benchmark for {model_path} with batch size:"
)
print(result.stderr)
all_model_succeed = False
continue
# Load and deserialize JSON results
if os.path.exists(json_output_file):
import json
with open(json_output_file, "r") as f:
json_data = json.load(f)
# Convert JSON data to BenchmarkResult objects
for data in json_data:
benchmark_result = BenchmarkResult(**data)
all_benchmark_results.append(benchmark_result)
benchmark_results.append(benchmark_result)
print(
f"Loaded {len(benchmark_results)} benchmark results from {json_output_file}"
)
# Clean up JSON file
os.remove(json_output_file)
else:
all_model_succeed = False
print(f"Warning: JSON output file {json_output_file} not found")
finally:
kill_process_tree(process.pid)
report_part = generate_markdown_report(PROFILE_DIR, benchmark_results)
self.full_report += report_part + "\n"
if is_in_ci():
write_github_step_summary(self.full_report)
if not all_model_succeed:
raise AssertionError("Some models failed the perf tests.")
if __name__ == "__main__":
unittest.main()

133
test/srt/test_nightly_text_models_perf.py
View File

@@ -1,133 +0,0 @@
import os
import subprocess
import time
import unittest
from sglang.bench_one_batch_server import BenchmarkResult, generate_markdown_report
from sglang.srt.utils import kill_process_tree
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
ModelLaunchSettings,
_parse_int_list_env,
is_in_ci,
parse_models,
popen_launch_server,
write_github_step_summary,
)
PROFILE_DIR = "performance_profiles_text_models"
class TestNightlyTextModelsPerformance(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.models = []
# TODO: replace with DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP1 or other model lists
for model_path in parse_models("meta-llama/Llama-3.1-8B-Instruct"):
cls.models.append(ModelLaunchSettings(model_path, tp_size=1))
for model_path in parse_models("Qwen/Qwen2-57B-A14B-Instruct"):
cls.models.append(ModelLaunchSettings(model_path, tp_size=2))
# (parse_models(DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP1), False, False),
# (parse_models(DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP2), False, True),
# (parse_models(DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP1), True, False),
# (parse_models(DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP2), True, True),
cls.base_url = DEFAULT_URL_FOR_TEST
cls.batch_sizes = [1, 1, 8, 16, 64]
cls.input_lens = tuple(_parse_int_list_env("NIGHTLY_INPUT_LENS", "4096"))
cls.output_lens = tuple(_parse_int_list_env("NIGHTLY_OUTPUT_LENS", "512"))
os.makedirs(PROFILE_DIR, exist_ok=True)
cls.full_report = f"## {cls.__name__}\n" + BenchmarkResult.help_str()
def test_bench_one_batch(self):
all_benchmark_results = []
all_model_succeed = True
for model_setup in self.models:
benchmark_results = []
with self.subTest(model=model_setup.model_path):
process = popen_launch_server(
model=model_setup.model_path,
base_url=self.base_url,
other_args=model_setup.extra_args,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
)
try:
profile_filename = (
f"{model_setup.model_path.replace('/', '_')}_{int(time.time())}"
)
profile_path_prefix = os.path.join(PROFILE_DIR, profile_filename)
json_output_file = f"results_{model_setup.model_path.replace('/', '_')}_{int(time.time())}.json"
command = [
"python3",
"-m",
"sglang.bench_one_batch_server",
"--model",
model_setup.model_path,
"--base-url",
self.base_url,
"--batch-size",
*[str(x) for x in self.batch_sizes],
"--input-len",
*[str(x) for x in self.input_lens],
"--output-len",
*[str(x) for x in self.output_lens],
"--show-report",
"--profile",
"--profile-by-stage",
"--profile-filename-prefix",
profile_path_prefix,
f"--output-path={json_output_file}",
"--no-append-to-github-summary",
]
print(f"Running command: {' '.join(command)}")
result = subprocess.run(command, capture_output=True, text=True)
if result.returncode != 0:
print(
f"Error running benchmark for {model_setup.model_path} with batch size:"
)
print(result.stderr)
# Continue to next batch size even if one fails
continue
# Load and deserialize JSON results
if os.path.exists(json_output_file):
import json
with open(json_output_file, "r") as f:
json_data = json.load(f)
# Convert JSON data to BenchmarkResult objects
for data in json_data:
benchmark_result = BenchmarkResult(**data)
all_benchmark_results.append(benchmark_result)
benchmark_results.append(benchmark_result)
print(
f"Loaded {len(benchmark_results)} benchmark results from {json_output_file}"
)
# Clean up JSON file
os.remove(json_output_file)
else:
all_model_succeed = False
print(f"Warning: JSON output file {json_output_file} not found")
finally:
kill_process_tree(process.pid)
report_part = generate_markdown_report(PROFILE_DIR, benchmark_results)
self.full_report += report_part + "\n"
if is_in_ci():
write_github_step_summary(self.full_report)
if not all_model_succeed:
raise AssertionError("Some models failed the perf tests.")
if __name__ == "__main__":
unittest.main()

158
test/srt/test_nightly_vlms_perf.py
View File

@@ -1,158 +0,0 @@
import os
import subprocess
import unittest
import warnings
from sglang.bench_one_batch_server import BenchmarkResult, generate_markdown_report
from sglang.srt.utils import kill_process_tree
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
ModelLaunchSettings,
_parse_int_list_env,
is_in_ci,
parse_models,
popen_launch_server,
write_github_step_summary,
)
PROFILE_DIR = "performance_profiles_vlms"
MODEL_DEFAULTS = [
# Keep conservative defaults. Can be overridden by env NIGHTLY_VLM_MODELS
ModelLaunchSettings(
"Qwen/Qwen2.5-VL-7B-Instruct",
extra_args=["--mem-fraction-static=0.7"],
),
ModelLaunchSettings(
"google/gemma-3-27b-it",
),
ModelLaunchSettings("Qwen/Qwen3-VL-30B-A3B-Instruct", extra_args=["--tp=2"]),
# "OpenGVLab/InternVL2_5-2B",
# buggy in official transformers impl
# "openbmb/MiniCPM-V-2_6",
]
class TestNightlyVLMModelsPerformance(unittest.TestCase):
@classmethod
def setUpClass(cls):
warnings.filterwarnings(
"ignore", category=ResourceWarning, message="unclosed.*socket"
)
nightly_vlm_models_str = os.environ.get("NIGHTLY_VLM_MODELS")
if nightly_vlm_models_str:
cls.models = []
model_paths = parse_models(nightly_vlm_models_str)
for model_path in model_paths:
cls.models.append(ModelLaunchSettings(model_path))
else:
cls.models = MODEL_DEFAULTS
cls.base_url = DEFAULT_URL_FOR_TEST
cls.batch_sizes = _parse_int_list_env("NIGHTLY_VLM_BATCH_SIZES", "1,1,2,8,16")
cls.input_lens = tuple(_parse_int_list_env("NIGHTLY_VLM_INPUT_LENS", "4096"))
cls.output_lens = tuple(_parse_int_list_env("NIGHTLY_VLM_OUTPUT_LENS", "512"))
cls.full_report = f"## {cls.__name__}\n" + BenchmarkResult.help_str()
def test_bench_one_batch(self):
all_benchmark_results = []
all_model_succeed = True
for model_setup in self.models:
benchmark_results = []
with self.subTest(model=model_setup.model_path):
process = popen_launch_server(
model=model_setup.model_path,
base_url=self.base_url,
other_args=model_setup.extra_args,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
)
try:
# Run bench_one_batch_server against the launched server
profile_filename = f"{model_setup.model_path.replace('/', '_')}"
# path for this run
profile_path_prefix = os.path.join(PROFILE_DIR, profile_filename)
# JSON output file for this model
json_output_file = (
f"results_{model_setup.model_path.replace('/', '_')}.json"
)
command = [
"python3",
"-m",
"sglang.bench_one_batch_server",
f"--model={model_setup.model_path}",
"--base-url",
self.base_url,
"--batch-size",
*[str(x) for x in self.batch_sizes],
"--input-len",
*[str(x) for x in self.input_lens],
"--output-len",
*[str(x) for x in self.output_lens],
"--trust-remote-code",
"--dataset-name=mmmu",
"--profile",
"--profile-by-stage",
f"--profile-filename-prefix={profile_path_prefix}",
"--show-report",
f"--output-path={json_output_file}",
"--no-append-to-github-summary",
]
print(f"Running command: {' '.join(command)}")
result = subprocess.run(command, capture_output=True, text=True)
if result.returncode != 0:
print(
f"Error running benchmark for {model_setup.model_path} with batch size:"
)
print(result.stderr)
continue
print(f"Output for {model_setup.model_path} with batch size:")
print(result.stdout)
# Load and deserialize JSON results
if os.path.exists(json_output_file):
import json
with open(json_output_file, "r") as f:
json_data = json.load(f)
# Convert JSON data to BenchmarkResult objects
for data in json_data:
benchmark_result = BenchmarkResult(**data)
all_benchmark_results.append(benchmark_result)
benchmark_results.append(benchmark_result)
print(
f"Loaded {len(benchmark_results)} benchmark results from {json_output_file}"
)
else:
all_model_succeed = False
print(f"Warning: JSON output file {json_output_file} not found")
finally:
kill_process_tree(process.pid)
report_part = generate_markdown_report(
PROFILE_DIR,
benchmark_results,
)
self.full_report += report_part + "\n"
if is_in_ci():
write_github_step_summary(self.full_report)
if not all_model_succeed:
raise AssertionError("Some models failed the perf tests.")
if __name__ == "__main__":
unittest.main()