chenchenghao/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 11 Packages Projects Releases Wiki Activity

Add Deepseek models into nightly tests (#12865)

Browse Source
This commit is contained in:
Kangyan-Zhou
2025-11-08 21:17:49 -08:00
committed by GitHub
parent f290e8016e
commit d134096319
18 changed files with 854 additions and 473 deletions
Download Patch File Download Diff File Expand all files Collapse all files

82
.github/workflows/nightly-test-b200.yml vendored Normal file
View File

@@ -0,0 +1,82 @@
name: Nightly Test
on:
schedule:
- cron: '0 0 * * *'
push:
branches:
- main
paths:
- "python/sglang/version.py"
workflow_dispatch:
concurrency:
group: nightly-test-${{ github.ref }}
cancel-in-progress: true
jobs:
nightly-test-4-gpu-b200:
if: github.repository == 'sgl-project/sglang'
runs-on: 4-gpu-b200
continue-on-error: true
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Install dependencies
run: |
IS_BLACKWELL=1 bash scripts/ci/ci_install_dependency.sh
- name: Run test
timeout-minutes: 60
run: |
cd test/srt
python3 run_suite.py --suite nightly-4-gpu-b200 --continue-on-error
nightly-test-8-gpu-b200:
if: github.repository == 'sgl-project/sglang'
runs-on: 8-gpu-b200
continue-on-error: true
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Install dependencies
run: |
IS_BLACKWELL=1 bash scripts/ci/ci_install_dependency.sh
- name: Run DeepSeek v3.1 nightly performance test
timeout-minutes: 180
env:
TRACE_BASE_URL: https://raw.githubusercontent.com/sglang-bot/sglang-ci-data/main/traces/${{ github.run_id }}
PERFETTO_RELAY_URL: ${{ vars.PERFETTO_RELAY_URL }}
run: |
rm -rf performance_profiles_deepseek_v31/
cd test/srt
IS_BLACKWELL=1 python3 nightly/test_deepseek_v31_perf.py --continue-on-error
- name: Publish DeepSeek v3.1 traces to storage repo
env:
GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }}
GITHUB_RUN_ID: ${{ github.run_id }}
GITHUB_RUN_NUMBER: ${{ github.run_number }}
run: |
python3 scripts/ci/publish_traces.py --traces-dir performance_profiles_deepseek_v31
- name: Run DeepSeek v3.2 nightly performance test
timeout-minutes: 180
env:
TRACE_BASE_URL: https://raw.githubusercontent.com/sglang-bot/sglang-ci-data/main/traces/${{ github.run_id }}
PERFETTO_RELAY_URL: ${{ vars.PERFETTO_RELAY_URL }}
run: |
rm -rf performance_profiles_deepseek_v32/
cd test/srt
IS_BLACKWELL=1 python3 nightly/test_deepseek_v32_perf.py --continue-on-error
- name: Publish DeepSeek v3.2 traces to storage repo
env:
GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }}
GITHUB_RUN_ID: ${{ github.run_id }}
GITHUB_RUN_NUMBER: ${{ github.run_number }}
run: |
python3 scripts/ci/publish_traces.py --traces-dir performance_profiles_deepseek_v32

30
.github/workflows/nightly-test.yml vendored
View File

@@ -31,7 +31,7 @@ jobs:
timeout-minutes: 120
run: |
cd test/srt
python3 test_nightly_text_models_gsm8k_eval.py
python3 nightly/test_text_models_gsm8k_eval.py
nightly-test-perf-text-models:
if: github.repository == 'sgl-project/sglang'
@@ -52,7 +52,7 @@ jobs:
PERFETTO_RELAY_URL: ${{ vars.PERFETTO_RELAY_URL }}
run: |
rm -rf test/srt/performance_profiles_text_models/
python3 test/srt/test_nightly_text_models_perf.py
python3 test/srt/nightly/test_text_models_perf.py
- name: Publish traces to storage repo
env:
@@ -60,7 +60,7 @@ jobs:
GITHUB_RUN_ID: ${{ github.run_id }}
GITHUB_RUN_NUMBER: ${{ github.run_number }}
run: |
python3 scripts/ci/publish_traces.py
python3 scripts/ci/publish_traces.py --traces-dir test/srt/performance_profiles_text_models
nightly-test-eval-vlms:
if: github.repository == 'sgl-project/sglang'
@@ -78,7 +78,7 @@ jobs:
timeout-minutes: 240
run: |
cd test/srt
python3 test_nightly_vlms_mmmu_eval.py
python3 nightly/test_vlms_mmmu_eval.py
nightly-test-perf-vlms:
if: github.repository == 'sgl-project/sglang'
@@ -99,7 +99,7 @@ jobs:
PERFETTO_RELAY_URL: ${{ vars.PERFETTO_RELAY_URL }}
run: |
rm -rf test/srt/performance_profiles_vlms/
python3 test/srt/test_nightly_vlms_perf.py
python3 test/srt/nightly/test_vlms_perf.py
- name: Publish traces to storage repo
env:
@@ -107,7 +107,7 @@ jobs:
GITHUB_RUN_ID: ${{ github.run_id }}
GITHUB_RUN_NUMBER: ${{ github.run_number }}
run: |
python3 scripts/ci/publish_traces.py --vlm
python3 scripts/ci/publish_traces.py --traces-dir test/srt/performance_profiles_vlms
nightly-test-1-gpu:
if: github.repository == 'sgl-project/sglang'
@@ -182,21 +182,3 @@ jobs:
run: |
cd test/srt
python3 run_suite.py --suite nightly-8-gpu-h20 --continue-on-error
nightly-test-4-gpu-b200:
if: github.repository == 'sgl-project/sglang'
runs-on: 4-gpu-b200
continue-on-error: true
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Install dependencies
run: |
IS_BLACKWELL=1 bash scripts/ci/ci_install_dependency.sh
- name: Run test
timeout-minutes: 60
run: |
cd test/srt
python3 run_suite.py --suite nightly-4-gpu-b200 --continue-on-error

6
python/sglang/srt/managers/scheduler_profiler_mixin.py
View File

@@ -139,7 +139,7 @@ class SchedulerProfilerMixin:
schema.writeSchema(connection)
connection.commit()
del connection
torch.distributed.barrier(self.tp_cpu_group)
torch.distributed.barrier(self.cpu_group)
self.rpd_profiler = rpdTracerControl()
self.rpd_profiler.setPythonTrace(True)
@@ -236,14 +236,14 @@ class SchedulerProfilerMixin:
self.torch_profiler.export_chrome_trace(
os.path.join(self.torch_profiler_output_dir, filename)
)
torch.distributed.barrier(self.tp_cpu_group)
torch.distributed.barrier(self.cpu_group)
if self.rpd_profiler is not None:
self.rpd_profiler.rangePop()
self.rpd_profiler.stop()
self.rpd_profiler.flush()
torch.distributed.barrier(self.tp_cpu_group)
torch.distributed.barrier(self.cpu_group)
if self.tp_rank == 0:
from sglang.srt.utils.rpd_utils import rpd_to_chrome_trace

26
scripts/ci/publish_traces.py
View File

@@ -145,7 +145,7 @@ def update_branch_ref(repo_owner, repo_name, branch, commit_sha, token):
make_github_request(url, token, method="PATCH", data=data)
def copy_trace_files(source_dir, target_base_path, is_vlm=False):
def copy_trace_files(source_dir, target_base_path):
"""Copy trace files and return list of files to upload"""
files_to_upload = []
@@ -171,7 +171,7 @@ def copy_trace_files(source_dir, target_base_path, is_vlm=False):
return files_to_upload
def publish_traces(traces_dir, run_id, run_number, is_vlm=False):
def publish_traces(traces_dir, run_id, run_number):
"""Publish traces to GitHub repository in a single commit"""
# Get environment variables
token = os.getenv("GITHUB_TOKEN")
@@ -186,7 +186,7 @@ def publish_traces(traces_dir, run_id, run_number, is_vlm=False):
target_base_path = f"traces/{run_id}"
# Copy trace files
files_to_upload = copy_trace_files(traces_dir, target_base_path, is_vlm)
files_to_upload = copy_trace_files(traces_dir, target_base_path)
if not files_to_upload:
print("No trace files found to upload")
@@ -261,11 +261,15 @@ def main():
parser = argparse.ArgumentParser(
description="Publish performance traces to GitHub repository"
)
parser.add_argument("--vlm", action="store_true", help="Process VLM model traces")
parser.add_argument(
"--traces-dir",
type=str,
required=True,
help="Traces directory to publish",
)
args = parser.parse_args()
# Get environment variables
run_id = os.getenv("GITHUB_RUN_ID", "test")
run_number = os.getenv("GITHUB_RUN_NUMBER", "12345")
@@ -275,16 +279,12 @@ def main():
)
sys.exit(1)
# Determine traces directory
if args.vlm:
traces_dir = "performance_profiles_vlms"
print("Processing VLM model traces")
else:
traces_dir = "performance_profiles_text_models"
print("Processing text model traces")
# Use traces directory
traces_dir = args.traces_dir
print(f"Processing traces from directory: {traces_dir}")
# Publish traces
publish_traces(traces_dir, run_id, run_number, args.vlm)
publish_traces(traces_dir, run_id, run_number)
if __name__ == "__main__":

3
test/README.md
View File

@@ -27,6 +27,7 @@ python3 test_choices.py
## Adding or Updating Tests in CI
- Create new test files under `test/srt` or `test/lang` depending on the type of test.
- For nightly tests, place them in `test/srt/nightly/`. Use the `NightlyBenchmarkRunner` helper class in `nightly_utils.py` for performance benchmarking tests.
- Ensure they are referenced in the respective `run_suite.py` (e.g., `test/srt/run_suite.py`) so they are picked up in CI. For most small test cases, they can be added to the `per-commit-1-gpu` suite. Sort the test cases alphabetically by name.
- Ensure you added `unittest.main()` for unittest and `pytest.main([__file__])` for pytest in the scripts. The CI run them via `python3 test_file.py`.
- The CI will run some suites such as `per-commit-1-gpu`, `per-commit-2-gpu`, and `nightly-1-gpu` automatically. If you need special setup or custom test groups, you may modify the workflows in [`.github/workflows/`](https://github.com/sgl-project/sglang/tree/main/.github/workflows).
@@ -46,4 +47,4 @@ python3 test_choices.py
## Adding New Models to Nightly CI
- **For text models**: extend [global model lists variables](https://github.com/sgl-project/sglang/blob/85c1f7937781199203b38bb46325a2840f353a04/python/sglang/test/test_utils.py#L104) in `test_utils.py`, or add more model lists
- **For vlms**: extend the `MODEL_THRESHOLDS` global dictionary in `test_nightly_vlms_.*.py`, see [here](https://github.com/sgl-project/sglang/blob/85c1f7937781199203b38bb46325a2840f353a04/test/srt/test_nightly_vlms_mmmu_eval.py#L19)
- **For vlms**: extend the `MODEL_THRESHOLDS` global dictionary in `test/srt/nightly/test_vlms_mmmu_eval.py`

291
test/srt/nightly/nightly_utils.py Normal file
View File

@@ -0,0 +1,291 @@
"""Utilities for running nightly performance benchmarks with profiling."""
import json
import os
import subprocess
import time
from typing import List, Optional, Tuple
from sglang.bench_one_batch_server import BenchmarkResult, generate_markdown_report
from sglang.srt.utils import kill_process_tree
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
is_in_ci,
popen_launch_server,
write_github_step_summary,
)
class NightlyBenchmarkRunner:
"""Helper class for running nightly performance benchmarks with profiling.
This class encapsulates common patterns used across nightly performance tests,
including profile directory management, benchmark command construction,
result parsing, and report generation.
"""
def __init__(
self,
profile_dir: str,
test_name: str,
base_url: str,
):
"""Initialize the benchmark runner.
Args:
profile_dir: Directory to store performance profiles
test_name: Name of the test (used for reporting)
base_url: Base URL for the server
"""
self.profile_dir = profile_dir
self.test_name = test_name
self.base_url = base_url
self.full_report = f"## {test_name}\n" + BenchmarkResult.help_str()
def setup_profile_directory(self) -> None:
"""Create the profile directory if it doesn't exist."""
os.makedirs(self.profile_dir, exist_ok=True)
def generate_profile_filename(
self, model_path: str, variant: str = ""
) -> Tuple[str, str]:
"""Generate unique profile filename and path for the model.
Args:
model_path: Path to the model (e.g., "deepseek-ai/DeepSeek-V3.1")
variant: Optional variant suffix (e.g., "basic", "mtp", "nsa")
Returns:
Tuple of (profile_path_prefix, json_output_file)
"""
timestamp = int(time.time())
model_safe_name = model_path.replace("/", "_")
# Build filename with optional variant
if variant:
profile_filename = f"{model_safe_name}_{variant}_{timestamp}"
json_filename = f"results_{model_safe_name}_{variant}_{timestamp}.json"
else:
profile_filename = f"{model_safe_name}_{timestamp}"
json_filename = f"results_{model_safe_name}_{timestamp}.json"
profile_path_prefix = os.path.join(self.profile_dir, profile_filename)
return profile_path_prefix, json_filename
def build_benchmark_command(
self,
model_path: str,
batch_sizes: List[int],
input_lens: Tuple[int, ...],
output_lens: Tuple[int, ...],
profile_path_prefix: str,
json_output_file: str,
extra_args: Optional[List[str]] = None,
) -> List[str]:
"""Build the benchmark command with all required arguments.
Args:
model_path: Path to the model
batch_sizes: List of batch sizes to test
input_lens: Tuple of input lengths to test
output_lens: Tuple of output lengths to test
profile_path_prefix: Prefix for profile output files
json_output_file: Path to JSON output file
extra_args: Optional extra arguments to append to command
Returns:
List of command arguments ready for subprocess.run()
"""
command = [
"python3",
"-m",
"sglang.bench_one_batch_server",
"--model",
model_path,
"--base-url",
self.base_url,
"--batch-size",
*[str(x) for x in batch_sizes],
"--input-len",
*[str(x) for x in input_lens],
"--output-len",
*[str(x) for x in output_lens],
"--show-report",
"--profile",
"--profile-by-stage",
"--profile-filename-prefix",
profile_path_prefix,
f"--output-path={json_output_file}",
"--no-append-to-github-summary",
]
if extra_args:
command.extend(extra_args)
return command
def run_benchmark_command(
self, command: List[str], model_description: str = ""
) -> Tuple[subprocess.CompletedProcess, bool]:
"""Execute the benchmark command and return the result.
Args:
command: Command to execute
model_description: Description for logging (e.g., "model_name (variant)")
Returns:
Tuple of (CompletedProcess, success_bool)
"""
print(f"Running command: {' '.join(command)}")
result = subprocess.run(command, capture_output=True, text=True)
if result.returncode != 0:
desc = model_description or "benchmark"
print(f"Error running benchmark for {desc}:")
print(result.stderr)
return result, False
return result, True
def load_benchmark_results(
self, json_output_file: str, model_description: str = ""
) -> Tuple[List[BenchmarkResult], bool]:
"""Load and parse benchmark results from JSON file.
Args:
json_output_file: Path to JSON output file
model_description: Description for logging
Returns:
Tuple of (list of BenchmarkResult objects, success_bool)
"""
benchmark_results = []
if not os.path.exists(json_output_file):
desc = model_description or "model"
print(f"Warning: JSON output file {json_output_file} not found for {desc}")
return benchmark_results, False
try:
with open(json_output_file, "r") as f:
json_data = json.load(f)
# Convert JSON data to BenchmarkResult objects
for data in json_data:
benchmark_result = BenchmarkResult(**data)
benchmark_results.append(benchmark_result)
print(
f"Loaded {len(benchmark_results)} benchmark results from {json_output_file}"
)
# Clean up JSON file
os.remove(json_output_file)
return benchmark_results, True
except Exception as e:
desc = model_description or "model"
print(f"Error loading benchmark results for {desc}: {e}")
# Try to clean up the file anyway
if os.path.exists(json_output_file):
os.remove(json_output_file)
return benchmark_results, False
def run_benchmark_for_model(
self,
model_path: str,
batch_sizes: List[int],
input_lens: Tuple[int, ...],
output_lens: Tuple[int, ...],
other_args: Optional[List[str]] = None,
variant: str = "",
extra_bench_args: Optional[List[str]] = None,
) -> Tuple[List[BenchmarkResult], bool]:
"""Run a complete benchmark for a single model with server management.
This method handles:
- Server launch and cleanup
- Profile filename generation
- Benchmark command construction and execution
- Result loading and parsing
Args:
model_path: Path to the model
batch_sizes: List of batch sizes to test
input_lens: Tuple of input lengths
output_lens: Tuple of output lengths
other_args: Arguments to pass to server launch
variant: Optional variant suffix (e.g., "basic", "mtp")
extra_bench_args: Extra arguments for the benchmark command
Returns:
Tuple of (list of BenchmarkResult objects, success_bool)
"""
benchmark_results = []
model_description = f"{model_path}" + (f" ({variant})" if variant else "")
# Launch server
process = popen_launch_server(
model=model_path,
base_url=self.base_url,
other_args=other_args or [],
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
)
try:
# Generate filenames
profile_path_prefix, json_output_file = self.generate_profile_filename(
model_path, variant
)
# Build and run benchmark command
command = self.build_benchmark_command(
model_path,
batch_sizes,
input_lens,
output_lens,
profile_path_prefix,
json_output_file,
extra_args=extra_bench_args,
)
result, cmd_success = self.run_benchmark_command(command, model_description)
if not cmd_success:
return benchmark_results, False
# Load results
benchmark_results, load_success = self.load_benchmark_results(
json_output_file, model_description
)
return benchmark_results, load_success
finally:
# Always clean up server process
kill_process_tree(process.pid)
def add_report(self, results: List[BenchmarkResult]) -> None:
"""Add benchmark results to the full report.
Args:
results: List of BenchmarkResult objects to add to report
"""
if results:
report_part = generate_markdown_report(self.profile_dir, results)
self.full_report += report_part + "\n"
def write_final_report(self) -> None:
"""Write the final report to GitHub summary if in CI."""
if is_in_ci():
write_github_step_summary(self.full_report)
def get_full_report(self) -> str:
"""Get the accumulated full report.
Returns:
The full markdown report as a string
"""
return self.full_report

98
test/srt/nightly/test_deepseek_v31_perf.py Normal file
View File

@@ -0,0 +1,98 @@
import unittest
from nightly_utils import NightlyBenchmarkRunner
from sglang.test.test_utils import DEFAULT_URL_FOR_TEST, _parse_int_list_env
DEEPSEEK_V31_MODEL_PATH = "deepseek-ai/DeepSeek-V3.1"
PROFILE_DIR = "performance_profiles_deepseek_v31"
class TestNightlyDeepseekV31Basic(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.model = DEEPSEEK_V31_MODEL_PATH
cls.base_url = DEFAULT_URL_FOR_TEST
cls.batch_sizes = [1, 1, 8, 16, 64]
cls.input_lens = tuple(_parse_int_list_env("NIGHTLY_INPUT_LENS", "4096"))
cls.output_lens = tuple(_parse_int_list_env("NIGHTLY_OUTPUT_LENS", "512"))
cls.other_args = [
"--trust-remote-code",
"--tp",
"8",
"--dp",
"8",
"--enable-dp-attention",
]
cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url)
cls.runner.setup_profile_directory()
def test_bench_one_batch(self):
results, success = self.runner.run_benchmark_for_model(
model_path=self.model,
batch_sizes=self.batch_sizes,
input_lens=self.input_lens,
output_lens=self.output_lens,
other_args=self.other_args,
variant="basic",
)
self.runner.add_report(results)
self.runner.write_final_report()
if not success:
raise AssertionError(
f"Benchmark failed for {self.model} with basic configuration"
)
class TestNightlyDeepseekV31MTP(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.model = DEEPSEEK_V31_MODEL_PATH
cls.base_url = DEFAULT_URL_FOR_TEST
cls.batch_sizes = [1, 1, 8, 16, 64]
cls.input_lens = tuple(_parse_int_list_env("NIGHTLY_INPUT_LENS", "4096"))
cls.output_lens = tuple(_parse_int_list_env("NIGHTLY_OUTPUT_LENS", "512"))
cls.other_args = [
"--trust-remote-code",
"--tp",
"8",
"--dp",
"8",
"--enable-dp-attention",
"--speculative-algorithm",
"EAGLE",
"--speculative-num-steps",
"3",
"--speculative-eagle-topk",
"1",
"--speculative-num-draft-tokens",
"4",
"--mem-frac",
"0.7",
]
cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url)
cls.runner.setup_profile_directory()
def test_bench_one_batch(self):
results, success = self.runner.run_benchmark_for_model(
model_path=self.model,
batch_sizes=self.batch_sizes,
input_lens=self.input_lens,
output_lens=self.output_lens,
other_args=self.other_args,
variant="mtp",
)
self.runner.add_report(results)
self.runner.write_final_report()
if not success:
raise AssertionError(
f"Benchmark failed for {self.model} with MTP configuration"
)
if __name__ == "__main__":
unittest.main()

142
test/srt/nightly/test_deepseek_v32_perf.py Normal file
View File

@@ -0,0 +1,142 @@
import unittest
from nightly_utils import NightlyBenchmarkRunner
from sglang.test.test_utils import DEFAULT_URL_FOR_TEST, _parse_int_list_env
DEEPSEEK_V32_MODEL_PATH = "deepseek-ai/DeepSeek-V3.2-Exp"
PROFILE_DIR = "performance_profiles_deepseek_v32"
class TestNightlyDeepseekV32Basic(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.model = DEEPSEEK_V32_MODEL_PATH
cls.base_url = DEFAULT_URL_FOR_TEST
cls.batch_sizes = [1, 1, 8, 16, 64]
cls.input_lens = tuple(_parse_int_list_env("NIGHTLY_INPUT_LENS", "4096"))
cls.output_lens = tuple(_parse_int_list_env("NIGHTLY_OUTPUT_LENS", "512"))
cls.other_args = [
"--trust-remote-code",
"--tp",
"8",
"--dp",
"8",
"--enable-dp-attention",
]
cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url)
cls.runner.setup_profile_directory()
def test_bench_one_batch(self):
results, success = self.runner.run_benchmark_for_model(
model_path=self.model,
batch_sizes=self.batch_sizes,
input_lens=self.input_lens,
output_lens=self.output_lens,
other_args=self.other_args,
variant="basic",
)
self.runner.add_report(results)
self.runner.write_final_report()
if not success:
raise AssertionError(
f"Benchmark failed for {self.model} with basic configuration"
)
class TestNightlyDeepseekV32MTP(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.model = DEEPSEEK_V32_MODEL_PATH
cls.base_url = DEFAULT_URL_FOR_TEST
cls.batch_sizes = [1, 1, 8, 16, 64]
cls.input_lens = tuple(_parse_int_list_env("NIGHTLY_INPUT_LENS", "4096"))
cls.output_lens = tuple(_parse_int_list_env("NIGHTLY_OUTPUT_LENS", "512"))
cls.other_args = [
"--trust-remote-code",
"--tp",
"8",
"--dp",
"8",
"--enable-dp-attention",
"--speculative-algorithm",
"EAGLE",
"--speculative-num-steps",
"3",
"--speculative-eagle-topk",
"1",
"--speculative-num-draft-tokens",
"4",
"--mem-frac",
"0.7",
]
cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url)
cls.runner.setup_profile_directory()
def test_bench_one_batch(self):
results, success = self.runner.run_benchmark_for_model(
model_path=self.model,
batch_sizes=self.batch_sizes,
input_lens=self.input_lens,
output_lens=self.output_lens,
other_args=self.other_args,
variant="mtp",
)
self.runner.add_report(results)
self.runner.write_final_report()
if not success:
raise AssertionError(
f"Benchmark failed for {self.model} with MTP configuration"
)
class TestNightlyDeepseekV32NSA(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.model = DEEPSEEK_V32_MODEL_PATH
cls.base_url = DEFAULT_URL_FOR_TEST
cls.batch_sizes = [1, 1, 8, 16, 64]
cls.input_lens = tuple(_parse_int_list_env("NIGHTLY_INPUT_LENS", "4096"))
cls.output_lens = tuple(_parse_int_list_env("NIGHTLY_OUTPUT_LENS", "512"))
cls.other_args = [
"--trust-remote-code",
"--tp",
"8",
"--dp",
"8",
"--enable-dp-attention",
"--attention-backend",
"nsa",
"--nsa-prefill-backend",
"flashmla_sparse",
"--nsa-decode-backend",
"flashmla_kv",
]
cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url)
cls.runner.setup_profile_directory()
def test_bench_one_batch(self):
results, success = self.runner.run_benchmark_for_model(
model_path=self.model,
batch_sizes=self.batch_sizes,
input_lens=self.input_lens,
output_lens=self.output_lens,
other_args=self.other_args,
variant="nsa",
)
self.runner.add_report(results)
self.runner.write_final_report()
if not success:
raise AssertionError(
f"Benchmark failed for {self.model} with NSA configuration"
)
if __name__ == "__main__":
unittest.main()

58
test/srt/nightly/test_gpt_oss_4gpu_perf.py Normal file
View File

@@ -0,0 +1,58 @@
import unittest
from nightly_utils import NightlyBenchmarkRunner
from sglang.test.test_utils import DEFAULT_URL_FOR_TEST
PROFILE_DIR = "performance_profiles_gpt_oss_4gpu"
class TestNightlyGptOss4GpuPerformance(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.models = [
(
"openai/gpt-oss-120b",
[
"--tp",
"4",
"--cuda-graph-max-bs",
"200",
"--mem-fraction-static",
"0.93",
],
),
]
cls.base_url = DEFAULT_URL_FOR_TEST
cls.batch_sizes = [1, 1, 8, 16, 64]
cls.input_lens = (4096,)
cls.output_lens = (512,)
cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url)
cls.runner.setup_profile_directory()
def test_bench_one_batch(self):
all_model_succeed = True
for model_path, other_args in self.models:
with self.subTest(model=model_path):
results, success = self.runner.run_benchmark_for_model(
model_path=model_path,
batch_sizes=self.batch_sizes,
input_lens=self.input_lens,
output_lens=self.output_lens,
other_args=other_args,
)
if not success:
all_model_succeed = False
self.runner.add_report(results)
self.runner.write_final_report()
if not all_model_succeed:
raise AssertionError("Some models failed the perf tests.")
if __name__ == "__main__":
unittest.main()

0
test/srt/test_nightly_gsm8k_eval_amd.py → test/srt/nightly/test_gsm8k_eval_amd.py
View File

0
test/srt/test_nightly_text_models_gsm8k_eval.py → test/srt/nightly/test_text_models_gsm8k_eval.py
View File

60
test/srt/nightly/test_text_models_perf.py Normal file
View File

@@ -0,0 +1,60 @@
import unittest
from nightly_utils import NightlyBenchmarkRunner
from sglang.test.test_utils import (
DEFAULT_URL_FOR_TEST,
ModelLaunchSettings,
_parse_int_list_env,
parse_models,
)
PROFILE_DIR = "performance_profiles_text_models"
class TestNightlyTextModelsPerformance(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.models = []
# TODO: replace with DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP1 or other model lists
for model_path in parse_models("meta-llama/Llama-3.1-8B-Instruct"):
cls.models.append(ModelLaunchSettings(model_path, tp_size=1))
for model_path in parse_models("Qwen/Qwen2-57B-A14B-Instruct"):
cls.models.append(ModelLaunchSettings(model_path, tp_size=2))
# (parse_models(DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP1), False, False),
# (parse_models(DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP2), False, True),
# (parse_models(DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP1), True, False),
# (parse_models(DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP2), True, True),
cls.base_url = DEFAULT_URL_FOR_TEST
cls.batch_sizes = [1, 1, 8, 16, 64]
cls.input_lens = tuple(_parse_int_list_env("NIGHTLY_INPUT_LENS", "4096"))
cls.output_lens = tuple(_parse_int_list_env("NIGHTLY_OUTPUT_LENS", "512"))
cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url)
cls.runner.setup_profile_directory()
def test_bench_one_batch(self):
all_model_succeed = True
for model_setup in self.models:
with self.subTest(model=model_setup.model_path):
results, success = self.runner.run_benchmark_for_model(
model_path=model_setup.model_path,
batch_sizes=self.batch_sizes,
input_lens=self.input_lens,
output_lens=self.output_lens,
other_args=model_setup.extra_args,
)
if not success:
all_model_succeed = False
self.runner.add_report(results)
self.runner.write_final_report()
if not all_model_succeed:
raise AssertionError("Some models failed the perf tests.")
if __name__ == "__main__":
unittest.main()

0
test/srt/test_nightly_vlms_mmmu_eval.py → test/srt/nightly/test_vlms_mmmu_eval.py
View File

88
test/srt/nightly/test_vlms_perf.py Normal file
View File

@@ -0,0 +1,88 @@
import os
import unittest
import warnings
from nightly_utils import NightlyBenchmarkRunner
from sglang.test.test_utils import (
DEFAULT_URL_FOR_TEST,
ModelLaunchSettings,
_parse_int_list_env,
parse_models,
)
PROFILE_DIR = "performance_profiles_vlms"
MODEL_DEFAULTS = [
# Keep conservative defaults. Can be overridden by env NIGHTLY_VLM_MODELS
ModelLaunchSettings(
"Qwen/Qwen2.5-VL-7B-Instruct",
extra_args=["--mem-fraction-static=0.7"],
),
ModelLaunchSettings(
"google/gemma-3-27b-it",
),
ModelLaunchSettings("Qwen/Qwen3-VL-30B-A3B-Instruct", extra_args=["--tp=2"]),
# "OpenGVLab/InternVL2_5-2B",
# buggy in official transformers impl
# "openbmb/MiniCPM-V-2_6",
]
class TestNightlyVLMModelsPerformance(unittest.TestCase):
@classmethod
def setUpClass(cls):
warnings.filterwarnings(
"ignore", category=ResourceWarning, message="unclosed.*socket"
)
nightly_vlm_models_str = os.environ.get("NIGHTLY_VLM_MODELS")
if nightly_vlm_models_str:
cls.models = []
model_paths = parse_models(nightly_vlm_models_str)
for model_path in model_paths:
cls.models.append(ModelLaunchSettings(model_path))
else:
cls.models = MODEL_DEFAULTS
cls.base_url = DEFAULT_URL_FOR_TEST
cls.batch_sizes = _parse_int_list_env("NIGHTLY_VLM_BATCH_SIZES", "1,1,2,8,16")
cls.input_lens = tuple(_parse_int_list_env("NIGHTLY_VLM_INPUT_LENS", "4096"))
cls.output_lens = tuple(_parse_int_list_env("NIGHTLY_VLM_OUTPUT_LENS", "512"))
cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url)
cls.runner.setup_profile_directory()
def test_bench_one_batch(self):
all_model_succeed = True
for model_setup in self.models:
with self.subTest(model=model_setup.model_path):
# VLMs need additional benchmark args for dataset and trust-remote-code
extra_bench_args = [
"--trust-remote-code",
"--dataset-name=mmmu",
]
results, success = self.runner.run_benchmark_for_model(
model_path=model_setup.model_path,
batch_sizes=self.batch_sizes,
input_lens=self.input_lens,
output_lens=self.output_lens,
other_args=model_setup.extra_args,
extra_bench_args=extra_bench_args,
)
if not success:
all_model_succeed = False
self.runner.add_report(results)
self.runner.write_final_report()
if not all_model_succeed:
raise AssertionError("Some models failed the perf tests.")
if __name__ == "__main__":
unittest.main()

18
test/srt/run_suite.py
View File

@@ -219,8 +219,9 @@ suites = {
],
"nightly-4-gpu-b200": [
TestFile("test_fp4_moe.py", 300),
TestFile("test_nightly_gpt_oss_4gpu_perf.py", 600),
TestFile("nightly/test_nightly_gpt_oss_4gpu_perf.py", 600),
],
"nightly-8-gpu-b200": [],
"nightly-4-gpu": [],
"nightly-8-gpu": [],
"nightly-8-gpu-h200": [],
@@ -334,11 +335,14 @@ suites = {
TestFile("test_moe_ep.py"),
TestFile("test_moe_eval_accuracy_large.py"),
TestFile("test_mscclpp.py"),
TestFile("test_nightly_gsm8k_eval.py"),
TestFile("test_nightly_text_models_gsm8k_eval.py"),
TestFile("test_nightly_text_models_perf.py"),
TestFile("test_nightly_vlms_mmmu_eval.py"),
TestFile("test_nightly_vlms_perf.py"),
TestFile("nightly/test_deepseek_v31_perf.py"),
TestFile("nightly/test_deepseek_v32_perf.py"),
TestFile("nightly/test_gpt_oss_4gpu_perf.py"),
TestFile("nightly/test_gsm8k_eval_amd.py"),
TestFile("nightly/test_text_models_gsm8k_eval.py"),
TestFile("nightly/test_text_models_perf.py"),
TestFile("nightly/test_vlms_mmmu_eval.py"),
TestFile("nightly/test_vlms_perf.py"),
TestFile("test_openai_adapter.py"),
TestFile("test_openai_function_calling.py"),
TestFile("test_openai_server.py"),
@@ -478,7 +482,7 @@ suite_amd = {
TestFile("test_deepseek_v3_mtp.py", 275),
],
"nightly-amd": [
TestFile("test_nightly_gsm8k_eval_amd.py"),
TestFile("nightly/test_gsm8k_eval_amd.py"),
],
}

134
test/srt/test_nightly_gpt_oss_4gpu_perf.py
View File

@@ -1,134 +0,0 @@
import os
import subprocess
import time
import unittest
from sglang.bench_one_batch_server import BenchmarkResult, generate_markdown_report
from sglang.srt.utils import kill_process_tree
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
is_in_ci,
popen_launch_server,
write_github_step_summary,
)
PROFILE_DIR = "performance_profiles_gpt_oss_4gpu"
class TestNightlyGptOss4GpuPerformance(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.models = [
("lmsys/gpt-oss-120b-bf16", ["--tp", "4", "--cuda-graph-max-bs", "200"]),
(
"openai/gpt-oss-120b",
[
"--tp",
"4",
"--cuda-graph-max-bs",
"200",
"--mem-fraction-static",
"0.93",
],
),
]
cls.base_url = DEFAULT_URL_FOR_TEST
cls.batch_sizes = [1, 1, 8, 16, 64]
cls.input_lens = (4096,)
cls.output_lens = (512,)
os.makedirs(PROFILE_DIR, exist_ok=True)
cls.full_report = f"## {cls.__name__}\n" + BenchmarkResult.help_str()
def test_bench_one_batch(self):
all_benchmark_results = []
all_model_succeed = True
for model_path, other_args in self.models:
benchmark_results = []
with self.subTest(model=model_path):
process = popen_launch_server(
model=model_path,
base_url=self.base_url,
other_args=other_args,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
)
try:
profile_filename = (
f"{model_path.replace('/', '_')}_{int(time.time())}"
)
profile_path_prefix = os.path.join(PROFILE_DIR, profile_filename)
json_output_file = f"results_{model_path.replace('/', '_')}_{int(time.time())}.json"
command = [
"python3",
"-m",
"sglang.bench_one_batch_server",
"--model",
model_path,
"--base-url",
self.base_url,
"--batch-size",
*[str(x) for x in self.batch_sizes],
"--input-len",
*[str(x) for x in self.input_lens],
"--output-len",
*[str(x) for x in self.output_lens],
"--show-report",
"--profile",
"--profile-by-stage",
"--profile-filename-prefix",
profile_path_prefix,
f"--output-path={json_output_file}",
"--no-append-to-github-summary",
]
print(f"Running command: {' '.join(command)}")
result = subprocess.run(command, capture_output=True, text=True)
if result.returncode != 0:
print(
f"Error running benchmark for {model_path} with batch size:"
)
print(result.stderr)
all_model_succeed = False
continue
# Load and deserialize JSON results
if os.path.exists(json_output_file):
import json
with open(json_output_file, "r") as f:
json_data = json.load(f)
# Convert JSON data to BenchmarkResult objects
for data in json_data:
benchmark_result = BenchmarkResult(**data)
all_benchmark_results.append(benchmark_result)
benchmark_results.append(benchmark_result)
print(
f"Loaded {len(benchmark_results)} benchmark results from {json_output_file}"
)
# Clean up JSON file
os.remove(json_output_file)
else:
all_model_succeed = False
print(f"Warning: JSON output file {json_output_file} not found")
finally:
kill_process_tree(process.pid)
report_part = generate_markdown_report(PROFILE_DIR, benchmark_results)
self.full_report += report_part + "\n"
if is_in_ci():
write_github_step_summary(self.full_report)
if not all_model_succeed:
raise AssertionError("Some models failed the perf tests.")
if __name__ == "__main__":
unittest.main()

133
test/srt/test_nightly_text_models_perf.py
View File

@@ -1,133 +0,0 @@
import os
import subprocess
import time
import unittest
from sglang.bench_one_batch_server import BenchmarkResult, generate_markdown_report
from sglang.srt.utils import kill_process_tree
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
ModelLaunchSettings,
_parse_int_list_env,
is_in_ci,
parse_models,
popen_launch_server,
write_github_step_summary,
)
PROFILE_DIR = "performance_profiles_text_models"
class TestNightlyTextModelsPerformance(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.models = []
# TODO: replace with DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP1 or other model lists
for model_path in parse_models("meta-llama/Llama-3.1-8B-Instruct"):
cls.models.append(ModelLaunchSettings(model_path, tp_size=1))
for model_path in parse_models("Qwen/Qwen2-57B-A14B-Instruct"):
cls.models.append(ModelLaunchSettings(model_path, tp_size=2))
# (parse_models(DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP1), False, False),
# (parse_models(DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP2), False, True),
# (parse_models(DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP1), True, False),
# (parse_models(DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP2), True, True),
cls.base_url = DEFAULT_URL_FOR_TEST
cls.batch_sizes = [1, 1, 8, 16, 64]
cls.input_lens = tuple(_parse_int_list_env("NIGHTLY_INPUT_LENS", "4096"))
cls.output_lens = tuple(_parse_int_list_env("NIGHTLY_OUTPUT_LENS", "512"))
os.makedirs(PROFILE_DIR, exist_ok=True)
cls.full_report = f"## {cls.__name__}\n" + BenchmarkResult.help_str()
def test_bench_one_batch(self):
all_benchmark_results = []
all_model_succeed = True
for model_setup in self.models:
benchmark_results = []
with self.subTest(model=model_setup.model_path):
process = popen_launch_server(
model=model_setup.model_path,
base_url=self.base_url,
other_args=model_setup.extra_args,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
)
try:
profile_filename = (
f"{model_setup.model_path.replace('/', '_')}_{int(time.time())}"
)
profile_path_prefix = os.path.join(PROFILE_DIR, profile_filename)
json_output_file = f"results_{model_setup.model_path.replace('/', '_')}_{int(time.time())}.json"
command = [
"python3",
"-m",
"sglang.bench_one_batch_server",
"--model",
model_setup.model_path,
"--base-url",
self.base_url,
"--batch-size",
*[str(x) for x in self.batch_sizes],
"--input-len",
*[str(x) for x in self.input_lens],
"--output-len",
*[str(x) for x in self.output_lens],
"--show-report",
"--profile",
"--profile-by-stage",
"--profile-filename-prefix",
profile_path_prefix,
f"--output-path={json_output_file}",
"--no-append-to-github-summary",
]
print(f"Running command: {' '.join(command)}")
result = subprocess.run(command, capture_output=True, text=True)
if result.returncode != 0:
print(
f"Error running benchmark for {model_setup.model_path} with batch size:"
)
print(result.stderr)
# Continue to next batch size even if one fails
continue
# Load and deserialize JSON results
if os.path.exists(json_output_file):
import json
with open(json_output_file, "r") as f:
json_data = json.load(f)
# Convert JSON data to BenchmarkResult objects
for data in json_data:
benchmark_result = BenchmarkResult(**data)
all_benchmark_results.append(benchmark_result)
benchmark_results.append(benchmark_result)
print(
f"Loaded {len(benchmark_results)} benchmark results from {json_output_file}"
)
# Clean up JSON file
os.remove(json_output_file)
else:
all_model_succeed = False
print(f"Warning: JSON output file {json_output_file} not found")
finally:
kill_process_tree(process.pid)
report_part = generate_markdown_report(PROFILE_DIR, benchmark_results)
self.full_report += report_part + "\n"
if is_in_ci():
write_github_step_summary(self.full_report)
if not all_model_succeed:
raise AssertionError("Some models failed the perf tests.")
if __name__ == "__main__":
unittest.main()

158
test/srt/test_nightly_vlms_perf.py
View File

@@ -1,158 +0,0 @@
import os
import subprocess
import unittest
import warnings
from sglang.bench_one_batch_server import BenchmarkResult, generate_markdown_report
from sglang.srt.utils import kill_process_tree
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
ModelLaunchSettings,
_parse_int_list_env,
is_in_ci,
parse_models,
popen_launch_server,
write_github_step_summary,
)
PROFILE_DIR = "performance_profiles_vlms"
MODEL_DEFAULTS = [
# Keep conservative defaults. Can be overridden by env NIGHTLY_VLM_MODELS
ModelLaunchSettings(
"Qwen/Qwen2.5-VL-7B-Instruct",
extra_args=["--mem-fraction-static=0.7"],
),
ModelLaunchSettings(
"google/gemma-3-27b-it",
),
ModelLaunchSettings("Qwen/Qwen3-VL-30B-A3B-Instruct", extra_args=["--tp=2"]),
# "OpenGVLab/InternVL2_5-2B",
# buggy in official transformers impl
# "openbmb/MiniCPM-V-2_6",
]
class TestNightlyVLMModelsPerformance(unittest.TestCase):
@classmethod
def setUpClass(cls):
warnings.filterwarnings(
"ignore", category=ResourceWarning, message="unclosed.*socket"
)
nightly_vlm_models_str = os.environ.get("NIGHTLY_VLM_MODELS")
if nightly_vlm_models_str:
cls.models = []
model_paths = parse_models(nightly_vlm_models_str)
for model_path in model_paths:
cls.models.append(ModelLaunchSettings(model_path))
else:
cls.models = MODEL_DEFAULTS
cls.base_url = DEFAULT_URL_FOR_TEST
cls.batch_sizes = _parse_int_list_env("NIGHTLY_VLM_BATCH_SIZES", "1,1,2,8,16")
cls.input_lens = tuple(_parse_int_list_env("NIGHTLY_VLM_INPUT_LENS", "4096"))
cls.output_lens = tuple(_parse_int_list_env("NIGHTLY_VLM_OUTPUT_LENS", "512"))
cls.full_report = f"## {cls.__name__}\n" + BenchmarkResult.help_str()
def test_bench_one_batch(self):
all_benchmark_results = []
all_model_succeed = True
for model_setup in self.models:
benchmark_results = []
with self.subTest(model=model_setup.model_path):
process = popen_launch_server(
model=model_setup.model_path,
base_url=self.base_url,
other_args=model_setup.extra_args,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
)
try:
# Run bench_one_batch_server against the launched server
profile_filename = f"{model_setup.model_path.replace('/', '_')}"
# path for this run
profile_path_prefix = os.path.join(PROFILE_DIR, profile_filename)
# JSON output file for this model
json_output_file = (
f"results_{model_setup.model_path.replace('/', '_')}.json"
)
command = [
"python3",
"-m",
"sglang.bench_one_batch_server",
f"--model={model_setup.model_path}",
"--base-url",
self.base_url,
"--batch-size",
*[str(x) for x in self.batch_sizes],
"--input-len",
*[str(x) for x in self.input_lens],
"--output-len",
*[str(x) for x in self.output_lens],
"--trust-remote-code",
"--dataset-name=mmmu",
"--profile",
"--profile-by-stage",
f"--profile-filename-prefix={profile_path_prefix}",
"--show-report",
f"--output-path={json_output_file}",
"--no-append-to-github-summary",
]
print(f"Running command: {' '.join(command)}")
result = subprocess.run(command, capture_output=True, text=True)
if result.returncode != 0:
print(
f"Error running benchmark for {model_setup.model_path} with batch size:"
)
print(result.stderr)
continue
print(f"Output for {model_setup.model_path} with batch size:")
print(result.stdout)
# Load and deserialize JSON results
if os.path.exists(json_output_file):
import json
with open(json_output_file, "r") as f:
json_data = json.load(f)
# Convert JSON data to BenchmarkResult objects
for data in json_data:
benchmark_result = BenchmarkResult(**data)
all_benchmark_results.append(benchmark_result)
benchmark_results.append(benchmark_result)
print(
f"Loaded {len(benchmark_results)} benchmark results from {json_output_file}"
)
else:
all_model_succeed = False
print(f"Warning: JSON output file {json_output_file} not found")
finally:
kill_process_tree(process.pid)
report_part = generate_markdown_report(
PROFILE_DIR,
benchmark_results,
)
self.full_report += report_part + "\n"
if is_in_ci():
write_github_step_summary(self.full_report)
if not all_model_succeed:
raise AssertionError("Some models failed the perf tests.")
if __name__ == "__main__":
unittest.main()