From d134096319cd8133cff0e6a77b66f2aac8ecf8db Mon Sep 17 00:00:00 2001 From: Kangyan-Zhou Date: Sat, 8 Nov 2025 21:17:49 -0800 Subject: [PATCH] Add Deepseek models into nightly tests (#12865) --- .github/workflows/nightly-test-b200.yml | 82 +++++ .github/workflows/nightly-test.yml | 30 +- .../srt/managers/scheduler_profiler_mixin.py | 6 +- scripts/ci/publish_traces.py | 26 +- test/README.md | 3 +- test/srt/nightly/nightly_utils.py | 291 ++++++++++++++++++ test/srt/nightly/test_deepseek_v31_perf.py | 98 ++++++ test/srt/nightly/test_deepseek_v32_perf.py | 142 +++++++++ test/srt/nightly/test_gpt_oss_4gpu_perf.py | 58 ++++ .../test_gsm8k_eval_amd.py} | 0 .../test_text_models_gsm8k_eval.py} | 0 test/srt/nightly/test_text_models_perf.py | 60 ++++ .../test_vlms_mmmu_eval.py} | 0 test/srt/nightly/test_vlms_perf.py | 88 ++++++ test/srt/run_suite.py | 18 +- test/srt/test_nightly_gpt_oss_4gpu_perf.py | 134 -------- test/srt/test_nightly_text_models_perf.py | 133 -------- test/srt/test_nightly_vlms_perf.py | 158 ---------- 18 files changed, 854 insertions(+), 473 deletions(-) create mode 100644 .github/workflows/nightly-test-b200.yml create mode 100644 test/srt/nightly/nightly_utils.py create mode 100644 test/srt/nightly/test_deepseek_v31_perf.py create mode 100644 test/srt/nightly/test_deepseek_v32_perf.py create mode 100644 test/srt/nightly/test_gpt_oss_4gpu_perf.py rename test/srt/{test_nightly_gsm8k_eval_amd.py => nightly/test_gsm8k_eval_amd.py} (100%) rename test/srt/{test_nightly_text_models_gsm8k_eval.py => nightly/test_text_models_gsm8k_eval.py} (100%) create mode 100644 test/srt/nightly/test_text_models_perf.py rename test/srt/{test_nightly_vlms_mmmu_eval.py => nightly/test_vlms_mmmu_eval.py} (100%) create mode 100644 test/srt/nightly/test_vlms_perf.py delete mode 100644 test/srt/test_nightly_gpt_oss_4gpu_perf.py delete mode 100644 test/srt/test_nightly_text_models_perf.py delete mode 100644 test/srt/test_nightly_vlms_perf.py diff --git a/.github/workflows/nightly-test-b200.yml b/.github/workflows/nightly-test-b200.yml new file mode 100644 index 000000000..a6d4d86f1 --- /dev/null +++ b/.github/workflows/nightly-test-b200.yml @@ -0,0 +1,82 @@ +name: Nightly Test + +on: + schedule: + - cron: '0 0 * * *' + push: + branches: + - main + paths: + - "python/sglang/version.py" + workflow_dispatch: + +concurrency: + group: nightly-test-${{ github.ref }} + cancel-in-progress: true + +jobs: + nightly-test-4-gpu-b200: + if: github.repository == 'sgl-project/sglang' + runs-on: 4-gpu-b200 + continue-on-error: true + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Install dependencies + run: | + IS_BLACKWELL=1 bash scripts/ci/ci_install_dependency.sh + + - name: Run test + timeout-minutes: 60 + run: | + cd test/srt + python3 run_suite.py --suite nightly-4-gpu-b200 --continue-on-error + + nightly-test-8-gpu-b200: + if: github.repository == 'sgl-project/sglang' + runs-on: 8-gpu-b200 + continue-on-error: true + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Install dependencies + run: | + IS_BLACKWELL=1 bash scripts/ci/ci_install_dependency.sh + + - name: Run DeepSeek v3.1 nightly performance test + timeout-minutes: 180 + env: + TRACE_BASE_URL: https://raw.githubusercontent.com/sglang-bot/sglang-ci-data/main/traces/${{ github.run_id }} + PERFETTO_RELAY_URL: ${{ vars.PERFETTO_RELAY_URL }} + run: | + rm -rf performance_profiles_deepseek_v31/ + cd test/srt + IS_BLACKWELL=1 python3 nightly/test_deepseek_v31_perf.py --continue-on-error + + - name: Publish DeepSeek v3.1 traces to storage repo + env: + GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }} + GITHUB_RUN_ID: ${{ github.run_id }} + GITHUB_RUN_NUMBER: ${{ github.run_number }} + run: | + python3 scripts/ci/publish_traces.py --traces-dir performance_profiles_deepseek_v31 + + - name: Run DeepSeek v3.2 nightly performance test + timeout-minutes: 180 + env: + TRACE_BASE_URL: https://raw.githubusercontent.com/sglang-bot/sglang-ci-data/main/traces/${{ github.run_id }} + PERFETTO_RELAY_URL: ${{ vars.PERFETTO_RELAY_URL }} + run: | + rm -rf performance_profiles_deepseek_v32/ + cd test/srt + IS_BLACKWELL=1 python3 nightly/test_deepseek_v32_perf.py --continue-on-error + + - name: Publish DeepSeek v3.2 traces to storage repo + env: + GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }} + GITHUB_RUN_ID: ${{ github.run_id }} + GITHUB_RUN_NUMBER: ${{ github.run_number }} + run: | + python3 scripts/ci/publish_traces.py --traces-dir performance_profiles_deepseek_v32 diff --git a/.github/workflows/nightly-test.yml b/.github/workflows/nightly-test.yml index 576e609c1..f9bebd1ea 100644 --- a/.github/workflows/nightly-test.yml +++ b/.github/workflows/nightly-test.yml @@ -31,7 +31,7 @@ jobs: timeout-minutes: 120 run: | cd test/srt - python3 test_nightly_text_models_gsm8k_eval.py + python3 nightly/test_text_models_gsm8k_eval.py nightly-test-perf-text-models: if: github.repository == 'sgl-project/sglang' @@ -52,7 +52,7 @@ jobs: PERFETTO_RELAY_URL: ${{ vars.PERFETTO_RELAY_URL }} run: | rm -rf test/srt/performance_profiles_text_models/ - python3 test/srt/test_nightly_text_models_perf.py + python3 test/srt/nightly/test_text_models_perf.py - name: Publish traces to storage repo env: @@ -60,7 +60,7 @@ jobs: GITHUB_RUN_ID: ${{ github.run_id }} GITHUB_RUN_NUMBER: ${{ github.run_number }} run: | - python3 scripts/ci/publish_traces.py + python3 scripts/ci/publish_traces.py --traces-dir test/srt/performance_profiles_text_models nightly-test-eval-vlms: if: github.repository == 'sgl-project/sglang' @@ -78,7 +78,7 @@ jobs: timeout-minutes: 240 run: | cd test/srt - python3 test_nightly_vlms_mmmu_eval.py + python3 nightly/test_vlms_mmmu_eval.py nightly-test-perf-vlms: if: github.repository == 'sgl-project/sglang' @@ -99,7 +99,7 @@ jobs: PERFETTO_RELAY_URL: ${{ vars.PERFETTO_RELAY_URL }} run: | rm -rf test/srt/performance_profiles_vlms/ - python3 test/srt/test_nightly_vlms_perf.py + python3 test/srt/nightly/test_vlms_perf.py - name: Publish traces to storage repo env: @@ -107,7 +107,7 @@ jobs: GITHUB_RUN_ID: ${{ github.run_id }} GITHUB_RUN_NUMBER: ${{ github.run_number }} run: | - python3 scripts/ci/publish_traces.py --vlm + python3 scripts/ci/publish_traces.py --traces-dir test/srt/performance_profiles_vlms nightly-test-1-gpu: if: github.repository == 'sgl-project/sglang' @@ -182,21 +182,3 @@ jobs: run: | cd test/srt python3 run_suite.py --suite nightly-8-gpu-h20 --continue-on-error - - nightly-test-4-gpu-b200: - if: github.repository == 'sgl-project/sglang' - runs-on: 4-gpu-b200 - continue-on-error: true - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Install dependencies - run: | - IS_BLACKWELL=1 bash scripts/ci/ci_install_dependency.sh - - - name: Run test - timeout-minutes: 60 - run: | - cd test/srt - python3 run_suite.py --suite nightly-4-gpu-b200 --continue-on-error diff --git a/python/sglang/srt/managers/scheduler_profiler_mixin.py b/python/sglang/srt/managers/scheduler_profiler_mixin.py index f9f3e1215..969158bd9 100644 --- a/python/sglang/srt/managers/scheduler_profiler_mixin.py +++ b/python/sglang/srt/managers/scheduler_profiler_mixin.py @@ -139,7 +139,7 @@ class SchedulerProfilerMixin: schema.writeSchema(connection) connection.commit() del connection - torch.distributed.barrier(self.tp_cpu_group) + torch.distributed.barrier(self.cpu_group) self.rpd_profiler = rpdTracerControl() self.rpd_profiler.setPythonTrace(True) @@ -236,14 +236,14 @@ class SchedulerProfilerMixin: self.torch_profiler.export_chrome_trace( os.path.join(self.torch_profiler_output_dir, filename) ) - torch.distributed.barrier(self.tp_cpu_group) + torch.distributed.barrier(self.cpu_group) if self.rpd_profiler is not None: self.rpd_profiler.rangePop() self.rpd_profiler.stop() self.rpd_profiler.flush() - torch.distributed.barrier(self.tp_cpu_group) + torch.distributed.barrier(self.cpu_group) if self.tp_rank == 0: from sglang.srt.utils.rpd_utils import rpd_to_chrome_trace diff --git a/scripts/ci/publish_traces.py b/scripts/ci/publish_traces.py index 5e3371c6c..735cc345f 100644 --- a/scripts/ci/publish_traces.py +++ b/scripts/ci/publish_traces.py @@ -145,7 +145,7 @@ def update_branch_ref(repo_owner, repo_name, branch, commit_sha, token): make_github_request(url, token, method="PATCH", data=data) -def copy_trace_files(source_dir, target_base_path, is_vlm=False): +def copy_trace_files(source_dir, target_base_path): """Copy trace files and return list of files to upload""" files_to_upload = [] @@ -171,7 +171,7 @@ def copy_trace_files(source_dir, target_base_path, is_vlm=False): return files_to_upload -def publish_traces(traces_dir, run_id, run_number, is_vlm=False): +def publish_traces(traces_dir, run_id, run_number): """Publish traces to GitHub repository in a single commit""" # Get environment variables token = os.getenv("GITHUB_TOKEN") @@ -186,7 +186,7 @@ def publish_traces(traces_dir, run_id, run_number, is_vlm=False): target_base_path = f"traces/{run_id}" # Copy trace files - files_to_upload = copy_trace_files(traces_dir, target_base_path, is_vlm) + files_to_upload = copy_trace_files(traces_dir, target_base_path) if not files_to_upload: print("No trace files found to upload") @@ -261,11 +261,15 @@ def main(): parser = argparse.ArgumentParser( description="Publish performance traces to GitHub repository" ) - parser.add_argument("--vlm", action="store_true", help="Process VLM model traces") + parser.add_argument( + "--traces-dir", + type=str, + required=True, + help="Traces directory to publish", + ) args = parser.parse_args() # Get environment variables - run_id = os.getenv("GITHUB_RUN_ID", "test") run_number = os.getenv("GITHUB_RUN_NUMBER", "12345") @@ -275,16 +279,12 @@ def main(): ) sys.exit(1) - # Determine traces directory - if args.vlm: - traces_dir = "performance_profiles_vlms" - print("Processing VLM model traces") - else: - traces_dir = "performance_profiles_text_models" - print("Processing text model traces") + # Use traces directory + traces_dir = args.traces_dir + print(f"Processing traces from directory: {traces_dir}") # Publish traces - publish_traces(traces_dir, run_id, run_number, args.vlm) + publish_traces(traces_dir, run_id, run_number) if __name__ == "__main__": diff --git a/test/README.md b/test/README.md index 1a6fd7c85..fa523bca2 100644 --- a/test/README.md +++ b/test/README.md @@ -27,6 +27,7 @@ python3 test_choices.py ## Adding or Updating Tests in CI - Create new test files under `test/srt` or `test/lang` depending on the type of test. +- For nightly tests, place them in `test/srt/nightly/`. Use the `NightlyBenchmarkRunner` helper class in `nightly_utils.py` for performance benchmarking tests. - Ensure they are referenced in the respective `run_suite.py` (e.g., `test/srt/run_suite.py`) so they are picked up in CI. For most small test cases, they can be added to the `per-commit-1-gpu` suite. Sort the test cases alphabetically by name. - Ensure you added `unittest.main()` for unittest and `pytest.main([__file__])` for pytest in the scripts. The CI run them via `python3 test_file.py`. - The CI will run some suites such as `per-commit-1-gpu`, `per-commit-2-gpu`, and `nightly-1-gpu` automatically. If you need special setup or custom test groups, you may modify the workflows in [`.github/workflows/`](https://github.com/sgl-project/sglang/tree/main/.github/workflows). @@ -46,4 +47,4 @@ python3 test_choices.py ## Adding New Models to Nightly CI - **For text models**: extend [global model lists variables](https://github.com/sgl-project/sglang/blob/85c1f7937781199203b38bb46325a2840f353a04/python/sglang/test/test_utils.py#L104) in `test_utils.py`, or add more model lists -- **For vlms**: extend the `MODEL_THRESHOLDS` global dictionary in `test_nightly_vlms_.*.py`, see [here](https://github.com/sgl-project/sglang/blob/85c1f7937781199203b38bb46325a2840f353a04/test/srt/test_nightly_vlms_mmmu_eval.py#L19) +- **For vlms**: extend the `MODEL_THRESHOLDS` global dictionary in `test/srt/nightly/test_vlms_mmmu_eval.py` diff --git a/test/srt/nightly/nightly_utils.py b/test/srt/nightly/nightly_utils.py new file mode 100644 index 000000000..e9c241a7f --- /dev/null +++ b/test/srt/nightly/nightly_utils.py @@ -0,0 +1,291 @@ +"""Utilities for running nightly performance benchmarks with profiling.""" + +import json +import os +import subprocess +import time +from typing import List, Optional, Tuple + +from sglang.bench_one_batch_server import BenchmarkResult, generate_markdown_report +from sglang.srt.utils import kill_process_tree +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + is_in_ci, + popen_launch_server, + write_github_step_summary, +) + + +class NightlyBenchmarkRunner: + """Helper class for running nightly performance benchmarks with profiling. + + This class encapsulates common patterns used across nightly performance tests, + including profile directory management, benchmark command construction, + result parsing, and report generation. + """ + + def __init__( + self, + profile_dir: str, + test_name: str, + base_url: str, + ): + """Initialize the benchmark runner. + + Args: + profile_dir: Directory to store performance profiles + test_name: Name of the test (used for reporting) + base_url: Base URL for the server + """ + self.profile_dir = profile_dir + self.test_name = test_name + self.base_url = base_url + self.full_report = f"## {test_name}\n" + BenchmarkResult.help_str() + + def setup_profile_directory(self) -> None: + """Create the profile directory if it doesn't exist.""" + os.makedirs(self.profile_dir, exist_ok=True) + + def generate_profile_filename( + self, model_path: str, variant: str = "" + ) -> Tuple[str, str]: + """Generate unique profile filename and path for the model. + + Args: + model_path: Path to the model (e.g., "deepseek-ai/DeepSeek-V3.1") + variant: Optional variant suffix (e.g., "basic", "mtp", "nsa") + + Returns: + Tuple of (profile_path_prefix, json_output_file) + """ + timestamp = int(time.time()) + model_safe_name = model_path.replace("/", "_") + + # Build filename with optional variant + if variant: + profile_filename = f"{model_safe_name}_{variant}_{timestamp}" + json_filename = f"results_{model_safe_name}_{variant}_{timestamp}.json" + else: + profile_filename = f"{model_safe_name}_{timestamp}" + json_filename = f"results_{model_safe_name}_{timestamp}.json" + + profile_path_prefix = os.path.join(self.profile_dir, profile_filename) + + return profile_path_prefix, json_filename + + def build_benchmark_command( + self, + model_path: str, + batch_sizes: List[int], + input_lens: Tuple[int, ...], + output_lens: Tuple[int, ...], + profile_path_prefix: str, + json_output_file: str, + extra_args: Optional[List[str]] = None, + ) -> List[str]: + """Build the benchmark command with all required arguments. + + Args: + model_path: Path to the model + batch_sizes: List of batch sizes to test + input_lens: Tuple of input lengths to test + output_lens: Tuple of output lengths to test + profile_path_prefix: Prefix for profile output files + json_output_file: Path to JSON output file + extra_args: Optional extra arguments to append to command + + Returns: + List of command arguments ready for subprocess.run() + """ + command = [ + "python3", + "-m", + "sglang.bench_one_batch_server", + "--model", + model_path, + "--base-url", + self.base_url, + "--batch-size", + *[str(x) for x in batch_sizes], + "--input-len", + *[str(x) for x in input_lens], + "--output-len", + *[str(x) for x in output_lens], + "--show-report", + "--profile", + "--profile-by-stage", + "--profile-filename-prefix", + profile_path_prefix, + f"--output-path={json_output_file}", + "--no-append-to-github-summary", + ] + + if extra_args: + command.extend(extra_args) + + return command + + def run_benchmark_command( + self, command: List[str], model_description: str = "" + ) -> Tuple[subprocess.CompletedProcess, bool]: + """Execute the benchmark command and return the result. + + Args: + command: Command to execute + model_description: Description for logging (e.g., "model_name (variant)") + + Returns: + Tuple of (CompletedProcess, success_bool) + """ + print(f"Running command: {' '.join(command)}") + result = subprocess.run(command, capture_output=True, text=True) + + if result.returncode != 0: + desc = model_description or "benchmark" + print(f"Error running benchmark for {desc}:") + print(result.stderr) + return result, False + + return result, True + + def load_benchmark_results( + self, json_output_file: str, model_description: str = "" + ) -> Tuple[List[BenchmarkResult], bool]: + """Load and parse benchmark results from JSON file. + + Args: + json_output_file: Path to JSON output file + model_description: Description for logging + + Returns: + Tuple of (list of BenchmarkResult objects, success_bool) + """ + benchmark_results = [] + + if not os.path.exists(json_output_file): + desc = model_description or "model" + print(f"Warning: JSON output file {json_output_file} not found for {desc}") + return benchmark_results, False + + try: + with open(json_output_file, "r") as f: + json_data = json.load(f) + + # Convert JSON data to BenchmarkResult objects + for data in json_data: + benchmark_result = BenchmarkResult(**data) + benchmark_results.append(benchmark_result) + + print( + f"Loaded {len(benchmark_results)} benchmark results from {json_output_file}" + ) + + # Clean up JSON file + os.remove(json_output_file) + + return benchmark_results, True + + except Exception as e: + desc = model_description or "model" + print(f"Error loading benchmark results for {desc}: {e}") + # Try to clean up the file anyway + if os.path.exists(json_output_file): + os.remove(json_output_file) + return benchmark_results, False + + def run_benchmark_for_model( + self, + model_path: str, + batch_sizes: List[int], + input_lens: Tuple[int, ...], + output_lens: Tuple[int, ...], + other_args: Optional[List[str]] = None, + variant: str = "", + extra_bench_args: Optional[List[str]] = None, + ) -> Tuple[List[BenchmarkResult], bool]: + """Run a complete benchmark for a single model with server management. + + This method handles: + - Server launch and cleanup + - Profile filename generation + - Benchmark command construction and execution + - Result loading and parsing + + Args: + model_path: Path to the model + batch_sizes: List of batch sizes to test + input_lens: Tuple of input lengths + output_lens: Tuple of output lengths + other_args: Arguments to pass to server launch + variant: Optional variant suffix (e.g., "basic", "mtp") + extra_bench_args: Extra arguments for the benchmark command + + Returns: + Tuple of (list of BenchmarkResult objects, success_bool) + """ + benchmark_results = [] + model_description = f"{model_path}" + (f" ({variant})" if variant else "") + + # Launch server + process = popen_launch_server( + model=model_path, + base_url=self.base_url, + other_args=other_args or [], + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + ) + + try: + # Generate filenames + profile_path_prefix, json_output_file = self.generate_profile_filename( + model_path, variant + ) + + # Build and run benchmark command + command = self.build_benchmark_command( + model_path, + batch_sizes, + input_lens, + output_lens, + profile_path_prefix, + json_output_file, + extra_args=extra_bench_args, + ) + + result, cmd_success = self.run_benchmark_command(command, model_description) + + if not cmd_success: + return benchmark_results, False + + # Load results + benchmark_results, load_success = self.load_benchmark_results( + json_output_file, model_description + ) + + return benchmark_results, load_success + + finally: + # Always clean up server process + kill_process_tree(process.pid) + + def add_report(self, results: List[BenchmarkResult]) -> None: + """Add benchmark results to the full report. + + Args: + results: List of BenchmarkResult objects to add to report + """ + if results: + report_part = generate_markdown_report(self.profile_dir, results) + self.full_report += report_part + "\n" + + def write_final_report(self) -> None: + """Write the final report to GitHub summary if in CI.""" + if is_in_ci(): + write_github_step_summary(self.full_report) + + def get_full_report(self) -> str: + """Get the accumulated full report. + + Returns: + The full markdown report as a string + """ + return self.full_report diff --git a/test/srt/nightly/test_deepseek_v31_perf.py b/test/srt/nightly/test_deepseek_v31_perf.py new file mode 100644 index 000000000..b1ceeaa99 --- /dev/null +++ b/test/srt/nightly/test_deepseek_v31_perf.py @@ -0,0 +1,98 @@ +import unittest + +from nightly_utils import NightlyBenchmarkRunner + +from sglang.test.test_utils import DEFAULT_URL_FOR_TEST, _parse_int_list_env + +DEEPSEEK_V31_MODEL_PATH = "deepseek-ai/DeepSeek-V3.1" +PROFILE_DIR = "performance_profiles_deepseek_v31" + + +class TestNightlyDeepseekV31Basic(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.model = DEEPSEEK_V31_MODEL_PATH + cls.base_url = DEFAULT_URL_FOR_TEST + cls.batch_sizes = [1, 1, 8, 16, 64] + cls.input_lens = tuple(_parse_int_list_env("NIGHTLY_INPUT_LENS", "4096")) + cls.output_lens = tuple(_parse_int_list_env("NIGHTLY_OUTPUT_LENS", "512")) + cls.other_args = [ + "--trust-remote-code", + "--tp", + "8", + "--dp", + "8", + "--enable-dp-attention", + ] + cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url) + cls.runner.setup_profile_directory() + + def test_bench_one_batch(self): + results, success = self.runner.run_benchmark_for_model( + model_path=self.model, + batch_sizes=self.batch_sizes, + input_lens=self.input_lens, + output_lens=self.output_lens, + other_args=self.other_args, + variant="basic", + ) + + self.runner.add_report(results) + self.runner.write_final_report() + + if not success: + raise AssertionError( + f"Benchmark failed for {self.model} with basic configuration" + ) + + +class TestNightlyDeepseekV31MTP(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.model = DEEPSEEK_V31_MODEL_PATH + cls.base_url = DEFAULT_URL_FOR_TEST + cls.batch_sizes = [1, 1, 8, 16, 64] + cls.input_lens = tuple(_parse_int_list_env("NIGHTLY_INPUT_LENS", "4096")) + cls.output_lens = tuple(_parse_int_list_env("NIGHTLY_OUTPUT_LENS", "512")) + cls.other_args = [ + "--trust-remote-code", + "--tp", + "8", + "--dp", + "8", + "--enable-dp-attention", + "--speculative-algorithm", + "EAGLE", + "--speculative-num-steps", + "3", + "--speculative-eagle-topk", + "1", + "--speculative-num-draft-tokens", + "4", + "--mem-frac", + "0.7", + ] + cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url) + cls.runner.setup_profile_directory() + + def test_bench_one_batch(self): + results, success = self.runner.run_benchmark_for_model( + model_path=self.model, + batch_sizes=self.batch_sizes, + input_lens=self.input_lens, + output_lens=self.output_lens, + other_args=self.other_args, + variant="mtp", + ) + + self.runner.add_report(results) + self.runner.write_final_report() + + if not success: + raise AssertionError( + f"Benchmark failed for {self.model} with MTP configuration" + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/nightly/test_deepseek_v32_perf.py b/test/srt/nightly/test_deepseek_v32_perf.py new file mode 100644 index 000000000..623d2d35b --- /dev/null +++ b/test/srt/nightly/test_deepseek_v32_perf.py @@ -0,0 +1,142 @@ +import unittest + +from nightly_utils import NightlyBenchmarkRunner + +from sglang.test.test_utils import DEFAULT_URL_FOR_TEST, _parse_int_list_env + +DEEPSEEK_V32_MODEL_PATH = "deepseek-ai/DeepSeek-V3.2-Exp" +PROFILE_DIR = "performance_profiles_deepseek_v32" + + +class TestNightlyDeepseekV32Basic(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.model = DEEPSEEK_V32_MODEL_PATH + cls.base_url = DEFAULT_URL_FOR_TEST + cls.batch_sizes = [1, 1, 8, 16, 64] + cls.input_lens = tuple(_parse_int_list_env("NIGHTLY_INPUT_LENS", "4096")) + cls.output_lens = tuple(_parse_int_list_env("NIGHTLY_OUTPUT_LENS", "512")) + cls.other_args = [ + "--trust-remote-code", + "--tp", + "8", + "--dp", + "8", + "--enable-dp-attention", + ] + cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url) + cls.runner.setup_profile_directory() + + def test_bench_one_batch(self): + results, success = self.runner.run_benchmark_for_model( + model_path=self.model, + batch_sizes=self.batch_sizes, + input_lens=self.input_lens, + output_lens=self.output_lens, + other_args=self.other_args, + variant="basic", + ) + + self.runner.add_report(results) + self.runner.write_final_report() + + if not success: + raise AssertionError( + f"Benchmark failed for {self.model} with basic configuration" + ) + + +class TestNightlyDeepseekV32MTP(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.model = DEEPSEEK_V32_MODEL_PATH + cls.base_url = DEFAULT_URL_FOR_TEST + cls.batch_sizes = [1, 1, 8, 16, 64] + cls.input_lens = tuple(_parse_int_list_env("NIGHTLY_INPUT_LENS", "4096")) + cls.output_lens = tuple(_parse_int_list_env("NIGHTLY_OUTPUT_LENS", "512")) + cls.other_args = [ + "--trust-remote-code", + "--tp", + "8", + "--dp", + "8", + "--enable-dp-attention", + "--speculative-algorithm", + "EAGLE", + "--speculative-num-steps", + "3", + "--speculative-eagle-topk", + "1", + "--speculative-num-draft-tokens", + "4", + "--mem-frac", + "0.7", + ] + cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url) + cls.runner.setup_profile_directory() + + def test_bench_one_batch(self): + results, success = self.runner.run_benchmark_for_model( + model_path=self.model, + batch_sizes=self.batch_sizes, + input_lens=self.input_lens, + output_lens=self.output_lens, + other_args=self.other_args, + variant="mtp", + ) + + self.runner.add_report(results) + self.runner.write_final_report() + + if not success: + raise AssertionError( + f"Benchmark failed for {self.model} with MTP configuration" + ) + + +class TestNightlyDeepseekV32NSA(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.model = DEEPSEEK_V32_MODEL_PATH + cls.base_url = DEFAULT_URL_FOR_TEST + cls.batch_sizes = [1, 1, 8, 16, 64] + cls.input_lens = tuple(_parse_int_list_env("NIGHTLY_INPUT_LENS", "4096")) + cls.output_lens = tuple(_parse_int_list_env("NIGHTLY_OUTPUT_LENS", "512")) + cls.other_args = [ + "--trust-remote-code", + "--tp", + "8", + "--dp", + "8", + "--enable-dp-attention", + "--attention-backend", + "nsa", + "--nsa-prefill-backend", + "flashmla_sparse", + "--nsa-decode-backend", + "flashmla_kv", + ] + cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url) + cls.runner.setup_profile_directory() + + def test_bench_one_batch(self): + results, success = self.runner.run_benchmark_for_model( + model_path=self.model, + batch_sizes=self.batch_sizes, + input_lens=self.input_lens, + output_lens=self.output_lens, + other_args=self.other_args, + variant="nsa", + ) + + self.runner.add_report(results) + self.runner.write_final_report() + + if not success: + raise AssertionError( + f"Benchmark failed for {self.model} with NSA configuration" + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/nightly/test_gpt_oss_4gpu_perf.py b/test/srt/nightly/test_gpt_oss_4gpu_perf.py new file mode 100644 index 000000000..5f8e26bdd --- /dev/null +++ b/test/srt/nightly/test_gpt_oss_4gpu_perf.py @@ -0,0 +1,58 @@ +import unittest + +from nightly_utils import NightlyBenchmarkRunner + +from sglang.test.test_utils import DEFAULT_URL_FOR_TEST + +PROFILE_DIR = "performance_profiles_gpt_oss_4gpu" + + +class TestNightlyGptOss4GpuPerformance(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.models = [ + ( + "openai/gpt-oss-120b", + [ + "--tp", + "4", + "--cuda-graph-max-bs", + "200", + "--mem-fraction-static", + "0.93", + ], + ), + ] + cls.base_url = DEFAULT_URL_FOR_TEST + cls.batch_sizes = [1, 1, 8, 16, 64] + cls.input_lens = (4096,) + cls.output_lens = (512,) + cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url) + cls.runner.setup_profile_directory() + + def test_bench_one_batch(self): + all_model_succeed = True + + for model_path, other_args in self.models: + with self.subTest(model=model_path): + results, success = self.runner.run_benchmark_for_model( + model_path=model_path, + batch_sizes=self.batch_sizes, + input_lens=self.input_lens, + output_lens=self.output_lens, + other_args=other_args, + ) + + if not success: + all_model_succeed = False + + self.runner.add_report(results) + + self.runner.write_final_report() + + if not all_model_succeed: + raise AssertionError("Some models failed the perf tests.") + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_nightly_gsm8k_eval_amd.py b/test/srt/nightly/test_gsm8k_eval_amd.py similarity index 100% rename from test/srt/test_nightly_gsm8k_eval_amd.py rename to test/srt/nightly/test_gsm8k_eval_amd.py diff --git a/test/srt/test_nightly_text_models_gsm8k_eval.py b/test/srt/nightly/test_text_models_gsm8k_eval.py similarity index 100% rename from test/srt/test_nightly_text_models_gsm8k_eval.py rename to test/srt/nightly/test_text_models_gsm8k_eval.py diff --git a/test/srt/nightly/test_text_models_perf.py b/test/srt/nightly/test_text_models_perf.py new file mode 100644 index 000000000..1e2cf70ff --- /dev/null +++ b/test/srt/nightly/test_text_models_perf.py @@ -0,0 +1,60 @@ +import unittest + +from nightly_utils import NightlyBenchmarkRunner + +from sglang.test.test_utils import ( + DEFAULT_URL_FOR_TEST, + ModelLaunchSettings, + _parse_int_list_env, + parse_models, +) + +PROFILE_DIR = "performance_profiles_text_models" + + +class TestNightlyTextModelsPerformance(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.models = [] + # TODO: replace with DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP1 or other model lists + for model_path in parse_models("meta-llama/Llama-3.1-8B-Instruct"): + cls.models.append(ModelLaunchSettings(model_path, tp_size=1)) + for model_path in parse_models("Qwen/Qwen2-57B-A14B-Instruct"): + cls.models.append(ModelLaunchSettings(model_path, tp_size=2)) + # (parse_models(DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP1), False, False), + # (parse_models(DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP2), False, True), + # (parse_models(DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP1), True, False), + # (parse_models(DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP2), True, True), + cls.base_url = DEFAULT_URL_FOR_TEST + cls.batch_sizes = [1, 1, 8, 16, 64] + cls.input_lens = tuple(_parse_int_list_env("NIGHTLY_INPUT_LENS", "4096")) + cls.output_lens = tuple(_parse_int_list_env("NIGHTLY_OUTPUT_LENS", "512")) + cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url) + cls.runner.setup_profile_directory() + + def test_bench_one_batch(self): + all_model_succeed = True + + for model_setup in self.models: + with self.subTest(model=model_setup.model_path): + results, success = self.runner.run_benchmark_for_model( + model_path=model_setup.model_path, + batch_sizes=self.batch_sizes, + input_lens=self.input_lens, + output_lens=self.output_lens, + other_args=model_setup.extra_args, + ) + + if not success: + all_model_succeed = False + + self.runner.add_report(results) + + self.runner.write_final_report() + + if not all_model_succeed: + raise AssertionError("Some models failed the perf tests.") + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_nightly_vlms_mmmu_eval.py b/test/srt/nightly/test_vlms_mmmu_eval.py similarity index 100% rename from test/srt/test_nightly_vlms_mmmu_eval.py rename to test/srt/nightly/test_vlms_mmmu_eval.py diff --git a/test/srt/nightly/test_vlms_perf.py b/test/srt/nightly/test_vlms_perf.py new file mode 100644 index 000000000..b837c0262 --- /dev/null +++ b/test/srt/nightly/test_vlms_perf.py @@ -0,0 +1,88 @@ +import os +import unittest +import warnings + +from nightly_utils import NightlyBenchmarkRunner + +from sglang.test.test_utils import ( + DEFAULT_URL_FOR_TEST, + ModelLaunchSettings, + _parse_int_list_env, + parse_models, +) + +PROFILE_DIR = "performance_profiles_vlms" + +MODEL_DEFAULTS = [ + # Keep conservative defaults. Can be overridden by env NIGHTLY_VLM_MODELS + ModelLaunchSettings( + "Qwen/Qwen2.5-VL-7B-Instruct", + extra_args=["--mem-fraction-static=0.7"], + ), + ModelLaunchSettings( + "google/gemma-3-27b-it", + ), + ModelLaunchSettings("Qwen/Qwen3-VL-30B-A3B-Instruct", extra_args=["--tp=2"]), + # "OpenGVLab/InternVL2_5-2B", + # buggy in official transformers impl + # "openbmb/MiniCPM-V-2_6", +] + + +class TestNightlyVLMModelsPerformance(unittest.TestCase): + @classmethod + def setUpClass(cls): + warnings.filterwarnings( + "ignore", category=ResourceWarning, message="unclosed.*socket" + ) + + nightly_vlm_models_str = os.environ.get("NIGHTLY_VLM_MODELS") + if nightly_vlm_models_str: + cls.models = [] + model_paths = parse_models(nightly_vlm_models_str) + for model_path in model_paths: + cls.models.append(ModelLaunchSettings(model_path)) + else: + cls.models = MODEL_DEFAULTS + + cls.base_url = DEFAULT_URL_FOR_TEST + + cls.batch_sizes = _parse_int_list_env("NIGHTLY_VLM_BATCH_SIZES", "1,1,2,8,16") + cls.input_lens = tuple(_parse_int_list_env("NIGHTLY_VLM_INPUT_LENS", "4096")) + cls.output_lens = tuple(_parse_int_list_env("NIGHTLY_VLM_OUTPUT_LENS", "512")) + cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url) + cls.runner.setup_profile_directory() + + def test_bench_one_batch(self): + all_model_succeed = True + + for model_setup in self.models: + with self.subTest(model=model_setup.model_path): + # VLMs need additional benchmark args for dataset and trust-remote-code + extra_bench_args = [ + "--trust-remote-code", + "--dataset-name=mmmu", + ] + + results, success = self.runner.run_benchmark_for_model( + model_path=model_setup.model_path, + batch_sizes=self.batch_sizes, + input_lens=self.input_lens, + output_lens=self.output_lens, + other_args=model_setup.extra_args, + extra_bench_args=extra_bench_args, + ) + + if not success: + all_model_succeed = False + + self.runner.add_report(results) + + self.runner.write_final_report() + + if not all_model_succeed: + raise AssertionError("Some models failed the perf tests.") + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/run_suite.py b/test/srt/run_suite.py index f1972a4cc..9e4460417 100644 --- a/test/srt/run_suite.py +++ b/test/srt/run_suite.py @@ -219,8 +219,9 @@ suites = { ], "nightly-4-gpu-b200": [ TestFile("test_fp4_moe.py", 300), - TestFile("test_nightly_gpt_oss_4gpu_perf.py", 600), + TestFile("nightly/test_nightly_gpt_oss_4gpu_perf.py", 600), ], + "nightly-8-gpu-b200": [], "nightly-4-gpu": [], "nightly-8-gpu": [], "nightly-8-gpu-h200": [], @@ -334,11 +335,14 @@ suites = { TestFile("test_moe_ep.py"), TestFile("test_moe_eval_accuracy_large.py"), TestFile("test_mscclpp.py"), - TestFile("test_nightly_gsm8k_eval.py"), - TestFile("test_nightly_text_models_gsm8k_eval.py"), - TestFile("test_nightly_text_models_perf.py"), - TestFile("test_nightly_vlms_mmmu_eval.py"), - TestFile("test_nightly_vlms_perf.py"), + TestFile("nightly/test_deepseek_v31_perf.py"), + TestFile("nightly/test_deepseek_v32_perf.py"), + TestFile("nightly/test_gpt_oss_4gpu_perf.py"), + TestFile("nightly/test_gsm8k_eval_amd.py"), + TestFile("nightly/test_text_models_gsm8k_eval.py"), + TestFile("nightly/test_text_models_perf.py"), + TestFile("nightly/test_vlms_mmmu_eval.py"), + TestFile("nightly/test_vlms_perf.py"), TestFile("test_openai_adapter.py"), TestFile("test_openai_function_calling.py"), TestFile("test_openai_server.py"), @@ -478,7 +482,7 @@ suite_amd = { TestFile("test_deepseek_v3_mtp.py", 275), ], "nightly-amd": [ - TestFile("test_nightly_gsm8k_eval_amd.py"), + TestFile("nightly/test_gsm8k_eval_amd.py"), ], } diff --git a/test/srt/test_nightly_gpt_oss_4gpu_perf.py b/test/srt/test_nightly_gpt_oss_4gpu_perf.py deleted file mode 100644 index 6c088f174..000000000 --- a/test/srt/test_nightly_gpt_oss_4gpu_perf.py +++ /dev/null @@ -1,134 +0,0 @@ -import os -import subprocess -import time -import unittest - -from sglang.bench_one_batch_server import BenchmarkResult, generate_markdown_report -from sglang.srt.utils import kill_process_tree -from sglang.test.test_utils import ( - DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - DEFAULT_URL_FOR_TEST, - is_in_ci, - popen_launch_server, - write_github_step_summary, -) - -PROFILE_DIR = "performance_profiles_gpt_oss_4gpu" - - -class TestNightlyGptOss4GpuPerformance(unittest.TestCase): - @classmethod - def setUpClass(cls): - cls.models = [ - ("lmsys/gpt-oss-120b-bf16", ["--tp", "4", "--cuda-graph-max-bs", "200"]), - ( - "openai/gpt-oss-120b", - [ - "--tp", - "4", - "--cuda-graph-max-bs", - "200", - "--mem-fraction-static", - "0.93", - ], - ), - ] - cls.base_url = DEFAULT_URL_FOR_TEST - cls.batch_sizes = [1, 1, 8, 16, 64] - cls.input_lens = (4096,) - cls.output_lens = (512,) - os.makedirs(PROFILE_DIR, exist_ok=True) - cls.full_report = f"## {cls.__name__}\n" + BenchmarkResult.help_str() - - def test_bench_one_batch(self): - all_benchmark_results = [] - all_model_succeed = True - for model_path, other_args in self.models: - benchmark_results = [] - with self.subTest(model=model_path): - process = popen_launch_server( - model=model_path, - base_url=self.base_url, - other_args=other_args, - timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - ) - try: - - profile_filename = ( - f"{model_path.replace('/', '_')}_{int(time.time())}" - ) - profile_path_prefix = os.path.join(PROFILE_DIR, profile_filename) - json_output_file = f"results_{model_path.replace('/', '_')}_{int(time.time())}.json" - - command = [ - "python3", - "-m", - "sglang.bench_one_batch_server", - "--model", - model_path, - "--base-url", - self.base_url, - "--batch-size", - *[str(x) for x in self.batch_sizes], - "--input-len", - *[str(x) for x in self.input_lens], - "--output-len", - *[str(x) for x in self.output_lens], - "--show-report", - "--profile", - "--profile-by-stage", - "--profile-filename-prefix", - profile_path_prefix, - f"--output-path={json_output_file}", - "--no-append-to-github-summary", - ] - - print(f"Running command: {' '.join(command)}") - result = subprocess.run(command, capture_output=True, text=True) - - if result.returncode != 0: - print( - f"Error running benchmark for {model_path} with batch size:" - ) - print(result.stderr) - all_model_succeed = False - continue - - # Load and deserialize JSON results - if os.path.exists(json_output_file): - import json - - with open(json_output_file, "r") as f: - json_data = json.load(f) - - # Convert JSON data to BenchmarkResult objects - for data in json_data: - benchmark_result = BenchmarkResult(**data) - all_benchmark_results.append(benchmark_result) - benchmark_results.append(benchmark_result) - - print( - f"Loaded {len(benchmark_results)} benchmark results from {json_output_file}" - ) - - # Clean up JSON file - os.remove(json_output_file) - else: - all_model_succeed = False - print(f"Warning: JSON output file {json_output_file} not found") - - finally: - kill_process_tree(process.pid) - - report_part = generate_markdown_report(PROFILE_DIR, benchmark_results) - self.full_report += report_part + "\n" - - if is_in_ci(): - write_github_step_summary(self.full_report) - - if not all_model_succeed: - raise AssertionError("Some models failed the perf tests.") - - -if __name__ == "__main__": - unittest.main() diff --git a/test/srt/test_nightly_text_models_perf.py b/test/srt/test_nightly_text_models_perf.py deleted file mode 100644 index 76250aea7..000000000 --- a/test/srt/test_nightly_text_models_perf.py +++ /dev/null @@ -1,133 +0,0 @@ -import os -import subprocess -import time -import unittest - -from sglang.bench_one_batch_server import BenchmarkResult, generate_markdown_report -from sglang.srt.utils import kill_process_tree -from sglang.test.test_utils import ( - DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - DEFAULT_URL_FOR_TEST, - ModelLaunchSettings, - _parse_int_list_env, - is_in_ci, - parse_models, - popen_launch_server, - write_github_step_summary, -) - -PROFILE_DIR = "performance_profiles_text_models" - - -class TestNightlyTextModelsPerformance(unittest.TestCase): - @classmethod - def setUpClass(cls): - cls.models = [] - # TODO: replace with DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP1 or other model lists - for model_path in parse_models("meta-llama/Llama-3.1-8B-Instruct"): - cls.models.append(ModelLaunchSettings(model_path, tp_size=1)) - for model_path in parse_models("Qwen/Qwen2-57B-A14B-Instruct"): - cls.models.append(ModelLaunchSettings(model_path, tp_size=2)) - # (parse_models(DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP1), False, False), - # (parse_models(DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP2), False, True), - # (parse_models(DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP1), True, False), - # (parse_models(DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP2), True, True), - cls.base_url = DEFAULT_URL_FOR_TEST - cls.batch_sizes = [1, 1, 8, 16, 64] - cls.input_lens = tuple(_parse_int_list_env("NIGHTLY_INPUT_LENS", "4096")) - cls.output_lens = tuple(_parse_int_list_env("NIGHTLY_OUTPUT_LENS", "512")) - os.makedirs(PROFILE_DIR, exist_ok=True) - cls.full_report = f"## {cls.__name__}\n" + BenchmarkResult.help_str() - - def test_bench_one_batch(self): - all_benchmark_results = [] - all_model_succeed = True - for model_setup in self.models: - benchmark_results = [] - with self.subTest(model=model_setup.model_path): - process = popen_launch_server( - model=model_setup.model_path, - base_url=self.base_url, - other_args=model_setup.extra_args, - timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - ) - try: - - profile_filename = ( - f"{model_setup.model_path.replace('/', '_')}_{int(time.time())}" - ) - profile_path_prefix = os.path.join(PROFILE_DIR, profile_filename) - json_output_file = f"results_{model_setup.model_path.replace('/', '_')}_{int(time.time())}.json" - - command = [ - "python3", - "-m", - "sglang.bench_one_batch_server", - "--model", - model_setup.model_path, - "--base-url", - self.base_url, - "--batch-size", - *[str(x) for x in self.batch_sizes], - "--input-len", - *[str(x) for x in self.input_lens], - "--output-len", - *[str(x) for x in self.output_lens], - "--show-report", - "--profile", - "--profile-by-stage", - "--profile-filename-prefix", - profile_path_prefix, - f"--output-path={json_output_file}", - "--no-append-to-github-summary", - ] - - print(f"Running command: {' '.join(command)}") - result = subprocess.run(command, capture_output=True, text=True) - - if result.returncode != 0: - print( - f"Error running benchmark for {model_setup.model_path} with batch size:" - ) - print(result.stderr) - # Continue to next batch size even if one fails - continue - - # Load and deserialize JSON results - if os.path.exists(json_output_file): - import json - - with open(json_output_file, "r") as f: - json_data = json.load(f) - - # Convert JSON data to BenchmarkResult objects - for data in json_data: - benchmark_result = BenchmarkResult(**data) - all_benchmark_results.append(benchmark_result) - benchmark_results.append(benchmark_result) - - print( - f"Loaded {len(benchmark_results)} benchmark results from {json_output_file}" - ) - - # Clean up JSON file - os.remove(json_output_file) - else: - all_model_succeed = False - print(f"Warning: JSON output file {json_output_file} not found") - - finally: - kill_process_tree(process.pid) - - report_part = generate_markdown_report(PROFILE_DIR, benchmark_results) - self.full_report += report_part + "\n" - - if is_in_ci(): - write_github_step_summary(self.full_report) - - if not all_model_succeed: - raise AssertionError("Some models failed the perf tests.") - - -if __name__ == "__main__": - unittest.main() diff --git a/test/srt/test_nightly_vlms_perf.py b/test/srt/test_nightly_vlms_perf.py deleted file mode 100644 index cf54bea16..000000000 --- a/test/srt/test_nightly_vlms_perf.py +++ /dev/null @@ -1,158 +0,0 @@ -import os -import subprocess -import unittest -import warnings - -from sglang.bench_one_batch_server import BenchmarkResult, generate_markdown_report -from sglang.srt.utils import kill_process_tree -from sglang.test.test_utils import ( - DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - DEFAULT_URL_FOR_TEST, - ModelLaunchSettings, - _parse_int_list_env, - is_in_ci, - parse_models, - popen_launch_server, - write_github_step_summary, -) - -PROFILE_DIR = "performance_profiles_vlms" - -MODEL_DEFAULTS = [ - # Keep conservative defaults. Can be overridden by env NIGHTLY_VLM_MODELS - ModelLaunchSettings( - "Qwen/Qwen2.5-VL-7B-Instruct", - extra_args=["--mem-fraction-static=0.7"], - ), - ModelLaunchSettings( - "google/gemma-3-27b-it", - ), - ModelLaunchSettings("Qwen/Qwen3-VL-30B-A3B-Instruct", extra_args=["--tp=2"]), - # "OpenGVLab/InternVL2_5-2B", - # buggy in official transformers impl - # "openbmb/MiniCPM-V-2_6", -] - - -class TestNightlyVLMModelsPerformance(unittest.TestCase): - @classmethod - def setUpClass(cls): - warnings.filterwarnings( - "ignore", category=ResourceWarning, message="unclosed.*socket" - ) - - nightly_vlm_models_str = os.environ.get("NIGHTLY_VLM_MODELS") - if nightly_vlm_models_str: - cls.models = [] - model_paths = parse_models(nightly_vlm_models_str) - for model_path in model_paths: - cls.models.append(ModelLaunchSettings(model_path)) - else: - cls.models = MODEL_DEFAULTS - - cls.base_url = DEFAULT_URL_FOR_TEST - - cls.batch_sizes = _parse_int_list_env("NIGHTLY_VLM_BATCH_SIZES", "1,1,2,8,16") - cls.input_lens = tuple(_parse_int_list_env("NIGHTLY_VLM_INPUT_LENS", "4096")) - cls.output_lens = tuple(_parse_int_list_env("NIGHTLY_VLM_OUTPUT_LENS", "512")) - cls.full_report = f"## {cls.__name__}\n" + BenchmarkResult.help_str() - - def test_bench_one_batch(self): - all_benchmark_results = [] - all_model_succeed = True - - for model_setup in self.models: - benchmark_results = [] - with self.subTest(model=model_setup.model_path): - process = popen_launch_server( - model=model_setup.model_path, - base_url=self.base_url, - other_args=model_setup.extra_args, - timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - ) - try: - # Run bench_one_batch_server against the launched server - profile_filename = f"{model_setup.model_path.replace('/', '_')}" - # path for this run - profile_path_prefix = os.path.join(PROFILE_DIR, profile_filename) - - # JSON output file for this model - json_output_file = ( - f"results_{model_setup.model_path.replace('/', '_')}.json" - ) - - command = [ - "python3", - "-m", - "sglang.bench_one_batch_server", - f"--model={model_setup.model_path}", - "--base-url", - self.base_url, - "--batch-size", - *[str(x) for x in self.batch_sizes], - "--input-len", - *[str(x) for x in self.input_lens], - "--output-len", - *[str(x) for x in self.output_lens], - "--trust-remote-code", - "--dataset-name=mmmu", - "--profile", - "--profile-by-stage", - f"--profile-filename-prefix={profile_path_prefix}", - "--show-report", - f"--output-path={json_output_file}", - "--no-append-to-github-summary", - ] - - print(f"Running command: {' '.join(command)}") - result = subprocess.run(command, capture_output=True, text=True) - - if result.returncode != 0: - print( - f"Error running benchmark for {model_setup.model_path} with batch size:" - ) - print(result.stderr) - continue - - print(f"Output for {model_setup.model_path} with batch size:") - print(result.stdout) - - # Load and deserialize JSON results - if os.path.exists(json_output_file): - import json - - with open(json_output_file, "r") as f: - json_data = json.load(f) - - # Convert JSON data to BenchmarkResult objects - for data in json_data: - benchmark_result = BenchmarkResult(**data) - all_benchmark_results.append(benchmark_result) - benchmark_results.append(benchmark_result) - - print( - f"Loaded {len(benchmark_results)} benchmark results from {json_output_file}" - ) - - else: - all_model_succeed = False - print(f"Warning: JSON output file {json_output_file} not found") - - finally: - kill_process_tree(process.pid) - - report_part = generate_markdown_report( - PROFILE_DIR, - benchmark_results, - ) - self.full_report += report_part + "\n" - - if is_in_ci(): - write_github_step_summary(self.full_report) - - if not all_model_succeed: - raise AssertionError("Some models failed the perf tests.") - - -if __name__ == "__main__": - unittest.main()