Add Deepseek models into nightly tests (#12865)
This commit is contained in:
Kangyan-Zhou
@@ -0,0 +1,291 @@
|
||||
"""Utilities for running nightly performance benchmarks with profiling."""
|
||||
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import time
|
||||
from typing import List, Optional, Tuple
|
||||
|
||||
from sglang.bench_one_batch_server import BenchmarkResult, generate_markdown_report
|
||||
from sglang.srt.utils import kill_process_tree
|
||||
from sglang.test.test_utils import (
|
||||
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||
is_in_ci,
|
||||
popen_launch_server,
|
||||
write_github_step_summary,
|
||||
)
|
||||
|
||||
|
||||
class NightlyBenchmarkRunner:
|
||||
"""Helper class for running nightly performance benchmarks with profiling.
|
||||
|
||||
This class encapsulates common patterns used across nightly performance tests,
|
||||
including profile directory management, benchmark command construction,
|
||||
result parsing, and report generation.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
profile_dir: str,
|
||||
test_name: str,
|
||||
base_url: str,
|
||||
):
|
||||
"""Initialize the benchmark runner.
|
||||
|
||||
Args:
|
||||
profile_dir: Directory to store performance profiles
|
||||
test_name: Name of the test (used for reporting)
|
||||
base_url: Base URL for the server
|
||||
"""
|
||||
self.profile_dir = profile_dir
|
||||
self.test_name = test_name
|
||||
self.base_url = base_url
|
||||
self.full_report = f"## {test_name}\n" + BenchmarkResult.help_str()
|
||||
|
||||
def setup_profile_directory(self) -> None:
|
||||
"""Create the profile directory if it doesn't exist."""
|
||||
os.makedirs(self.profile_dir, exist_ok=True)
|
||||
|
||||
def generate_profile_filename(
|
||||
self, model_path: str, variant: str = ""
|
||||
) -> Tuple[str, str]:
|
||||
"""Generate unique profile filename and path for the model.
|
||||
|
||||
Args:
|
||||
model_path: Path to the model (e.g., "deepseek-ai/DeepSeek-V3.1")
|
||||
variant: Optional variant suffix (e.g., "basic", "mtp", "nsa")
|
||||
|
||||
Returns:
|
||||
Tuple of (profile_path_prefix, json_output_file)
|
||||
"""
|
||||
timestamp = int(time.time())
|
||||
model_safe_name = model_path.replace("/", "_")
|
||||
|
||||
# Build filename with optional variant
|
||||
if variant:
|
||||
profile_filename = f"{model_safe_name}_{variant}_{timestamp}"
|
||||
json_filename = f"results_{model_safe_name}_{variant}_{timestamp}.json"
|
||||
else:
|
||||
profile_filename = f"{model_safe_name}_{timestamp}"
|
||||
json_filename = f"results_{model_safe_name}_{timestamp}.json"
|
||||
|
||||
profile_path_prefix = os.path.join(self.profile_dir, profile_filename)
|
||||
|
||||
return profile_path_prefix, json_filename
|
||||
|
||||
def build_benchmark_command(
|
||||
self,
|
||||
model_path: str,
|
||||
batch_sizes: List[int],
|
||||
input_lens: Tuple[int, ...],
|
||||
output_lens: Tuple[int, ...],
|
||||
profile_path_prefix: str,
|
||||
json_output_file: str,
|
||||
extra_args: Optional[List[str]] = None,
|
||||
) -> List[str]:
|
||||
"""Build the benchmark command with all required arguments.
|
||||
|
||||
Args:
|
||||
model_path: Path to the model
|
||||
batch_sizes: List of batch sizes to test
|
||||
input_lens: Tuple of input lengths to test
|
||||
output_lens: Tuple of output lengths to test
|
||||
profile_path_prefix: Prefix for profile output files
|
||||
json_output_file: Path to JSON output file
|
||||
extra_args: Optional extra arguments to append to command
|
||||
|
||||
Returns:
|
||||
List of command arguments ready for subprocess.run()
|
||||
"""
|
||||
command = [
|
||||
"python3",
|
||||
"-m",
|
||||
"sglang.bench_one_batch_server",
|
||||
"--model",
|
||||
model_path,
|
||||
"--base-url",
|
||||
self.base_url,
|
||||
"--batch-size",
|
||||
*[str(x) for x in batch_sizes],
|
||||
"--input-len",
|
||||
*[str(x) for x in input_lens],
|
||||
"--output-len",
|
||||
*[str(x) for x in output_lens],
|
||||
"--show-report",
|
||||
"--profile",
|
||||
"--profile-by-stage",
|
||||
"--profile-filename-prefix",
|
||||
profile_path_prefix,
|
||||
f"--output-path={json_output_file}",
|
||||
"--no-append-to-github-summary",
|
||||
]
|
||||
|
||||
if extra_args:
|
||||
command.extend(extra_args)
|
||||
|
||||
return command
|
||||
|
||||
def run_benchmark_command(
|
||||
self, command: List[str], model_description: str = ""
|
||||
) -> Tuple[subprocess.CompletedProcess, bool]:
|
||||
"""Execute the benchmark command and return the result.
|
||||
|
||||
Args:
|
||||
command: Command to execute
|
||||
model_description: Description for logging (e.g., "model_name (variant)")
|
||||
|
||||
Returns:
|
||||
Tuple of (CompletedProcess, success_bool)
|
||||
"""
|
||||
print(f"Running command: {' '.join(command)}")
|
||||
result = subprocess.run(command, capture_output=True, text=True)
|
||||
|
||||
if result.returncode != 0:
|
||||
desc = model_description or "benchmark"
|
||||
print(f"Error running benchmark for {desc}:")
|
||||
print(result.stderr)
|
||||
return result, False
|
||||
|
||||
return result, True
|
||||
|
||||
def load_benchmark_results(
|
||||
self, json_output_file: str, model_description: str = ""
|
||||
) -> Tuple[List[BenchmarkResult], bool]:
|
||||
"""Load and parse benchmark results from JSON file.
|
||||
|
||||
Args:
|
||||
json_output_file: Path to JSON output file
|
||||
model_description: Description for logging
|
||||
|
||||
Returns:
|
||||
Tuple of (list of BenchmarkResult objects, success_bool)
|
||||
"""
|
||||
benchmark_results = []
|
||||
|
||||
if not os.path.exists(json_output_file):
|
||||
desc = model_description or "model"
|
||||
print(f"Warning: JSON output file {json_output_file} not found for {desc}")
|
||||
return benchmark_results, False
|
||||
|
||||
try:
|
||||
with open(json_output_file, "r") as f:
|
||||
json_data = json.load(f)
|
||||
|
||||
# Convert JSON data to BenchmarkResult objects
|
||||
for data in json_data:
|
||||
benchmark_result = BenchmarkResult(**data)
|
||||
benchmark_results.append(benchmark_result)
|
||||
|
||||
print(
|
||||
f"Loaded {len(benchmark_results)} benchmark results from {json_output_file}"
|
||||
)
|
||||
|
||||
# Clean up JSON file
|
||||
os.remove(json_output_file)
|
||||
|
||||
return benchmark_results, True
|
||||
|
||||
except Exception as e:
|
||||
desc = model_description or "model"
|
||||
print(f"Error loading benchmark results for {desc}: {e}")
|
||||
# Try to clean up the file anyway
|
||||
if os.path.exists(json_output_file):
|
||||
os.remove(json_output_file)
|
||||
return benchmark_results, False
|
||||
|
||||
def run_benchmark_for_model(
|
||||
self,
|
||||
model_path: str,
|
||||
batch_sizes: List[int],
|
||||
input_lens: Tuple[int, ...],
|
||||
output_lens: Tuple[int, ...],
|
||||
other_args: Optional[List[str]] = None,
|
||||
variant: str = "",
|
||||
extra_bench_args: Optional[List[str]] = None,
|
||||
) -> Tuple[List[BenchmarkResult], bool]:
|
||||
"""Run a complete benchmark for a single model with server management.
|
||||
|
||||
This method handles:
|
||||
- Server launch and cleanup
|
||||
- Profile filename generation
|
||||
- Benchmark command construction and execution
|
||||
- Result loading and parsing
|
||||
|
||||
Args:
|
||||
model_path: Path to the model
|
||||
batch_sizes: List of batch sizes to test
|
||||
input_lens: Tuple of input lengths
|
||||
output_lens: Tuple of output lengths
|
||||
other_args: Arguments to pass to server launch
|
||||
variant: Optional variant suffix (e.g., "basic", "mtp")
|
||||
extra_bench_args: Extra arguments for the benchmark command
|
||||
|
||||
Returns:
|
||||
Tuple of (list of BenchmarkResult objects, success_bool)
|
||||
"""
|
||||
benchmark_results = []
|
||||
model_description = f"{model_path}" + (f" ({variant})" if variant else "")
|
||||
|
||||
# Launch server
|
||||
process = popen_launch_server(
|
||||
model=model_path,
|
||||
base_url=self.base_url,
|
||||
other_args=other_args or [],
|
||||
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||
)
|
||||
|
||||
try:
|
||||
# Generate filenames
|
||||
profile_path_prefix, json_output_file = self.generate_profile_filename(
|
||||
model_path, variant
|
||||
)
|
||||
|
||||
# Build and run benchmark command
|
||||
command = self.build_benchmark_command(
|
||||
model_path,
|
||||
batch_sizes,
|
||||
input_lens,
|
||||
output_lens,
|
||||
profile_path_prefix,
|
||||
json_output_file,
|
||||
extra_args=extra_bench_args,
|
||||
)
|
||||
|
||||
result, cmd_success = self.run_benchmark_command(command, model_description)
|
||||
|
||||
if not cmd_success:
|
||||
return benchmark_results, False
|
||||
|
||||
# Load results
|
||||
benchmark_results, load_success = self.load_benchmark_results(
|
||||
json_output_file, model_description
|
||||
)
|
||||
|
||||
return benchmark_results, load_success
|
||||
|
||||
finally:
|
||||
# Always clean up server process
|
||||
kill_process_tree(process.pid)
|
||||
|
||||
def add_report(self, results: List[BenchmarkResult]) -> None:
|
||||
"""Add benchmark results to the full report.
|
||||
|
||||
Args:
|
||||
results: List of BenchmarkResult objects to add to report
|
||||
"""
|
||||
if results:
|
||||
report_part = generate_markdown_report(self.profile_dir, results)
|
||||
self.full_report += report_part + "\n"
|
||||
|
||||
def write_final_report(self) -> None:
|
||||
"""Write the final report to GitHub summary if in CI."""
|
||||
if is_in_ci():
|
||||
write_github_step_summary(self.full_report)
|
||||
|
||||
def get_full_report(self) -> str:
|
||||
"""Get the accumulated full report.
|
||||
|
||||
Returns:
|
||||
The full markdown report as a string
|
||||
"""
|
||||
return self.full_report
|
||||
@@ -0,0 +1,98 @@
|
||||
import unittest
|
||||
|
||||
from nightly_utils import NightlyBenchmarkRunner
|
||||
|
||||
from sglang.test.test_utils import DEFAULT_URL_FOR_TEST, _parse_int_list_env
|
||||
|
||||
DEEPSEEK_V31_MODEL_PATH = "deepseek-ai/DeepSeek-V3.1"
|
||||
PROFILE_DIR = "performance_profiles_deepseek_v31"
|
||||
|
||||
|
||||
class TestNightlyDeepseekV31Basic(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.model = DEEPSEEK_V31_MODEL_PATH
|
||||
cls.base_url = DEFAULT_URL_FOR_TEST
|
||||
cls.batch_sizes = [1, 1, 8, 16, 64]
|
||||
cls.input_lens = tuple(_parse_int_list_env("NIGHTLY_INPUT_LENS", "4096"))
|
||||
cls.output_lens = tuple(_parse_int_list_env("NIGHTLY_OUTPUT_LENS", "512"))
|
||||
cls.other_args = [
|
||||
"--trust-remote-code",
|
||||
"--tp",
|
||||
"8",
|
||||
"--dp",
|
||||
"8",
|
||||
"--enable-dp-attention",
|
||||
]
|
||||
cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url)
|
||||
cls.runner.setup_profile_directory()
|
||||
|
||||
def test_bench_one_batch(self):
|
||||
results, success = self.runner.run_benchmark_for_model(
|
||||
model_path=self.model,
|
||||
batch_sizes=self.batch_sizes,
|
||||
input_lens=self.input_lens,
|
||||
output_lens=self.output_lens,
|
||||
other_args=self.other_args,
|
||||
variant="basic",
|
||||
)
|
||||
|
||||
self.runner.add_report(results)
|
||||
self.runner.write_final_report()
|
||||
|
||||
if not success:
|
||||
raise AssertionError(
|
||||
f"Benchmark failed for {self.model} with basic configuration"
|
||||
)
|
||||
|
||||
|
||||
class TestNightlyDeepseekV31MTP(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.model = DEEPSEEK_V31_MODEL_PATH
|
||||
cls.base_url = DEFAULT_URL_FOR_TEST
|
||||
cls.batch_sizes = [1, 1, 8, 16, 64]
|
||||
cls.input_lens = tuple(_parse_int_list_env("NIGHTLY_INPUT_LENS", "4096"))
|
||||
cls.output_lens = tuple(_parse_int_list_env("NIGHTLY_OUTPUT_LENS", "512"))
|
||||
cls.other_args = [
|
||||
"--trust-remote-code",
|
||||
"--tp",
|
||||
"8",
|
||||
"--dp",
|
||||
"8",
|
||||
"--enable-dp-attention",
|
||||
"--speculative-algorithm",
|
||||
"EAGLE",
|
||||
"--speculative-num-steps",
|
||||
"3",
|
||||
"--speculative-eagle-topk",
|
||||
"1",
|
||||
"--speculative-num-draft-tokens",
|
||||
"4",
|
||||
"--mem-frac",
|
||||
"0.7",
|
||||
]
|
||||
cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url)
|
||||
cls.runner.setup_profile_directory()
|
||||
|
||||
def test_bench_one_batch(self):
|
||||
results, success = self.runner.run_benchmark_for_model(
|
||||
model_path=self.model,
|
||||
batch_sizes=self.batch_sizes,
|
||||
input_lens=self.input_lens,
|
||||
output_lens=self.output_lens,
|
||||
other_args=self.other_args,
|
||||
variant="mtp",
|
||||
)
|
||||
|
||||
self.runner.add_report(results)
|
||||
self.runner.write_final_report()
|
||||
|
||||
if not success:
|
||||
raise AssertionError(
|
||||
f"Benchmark failed for {self.model} with MTP configuration"
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,142 @@
|
||||
import unittest
|
||||
|
||||
from nightly_utils import NightlyBenchmarkRunner
|
||||
|
||||
from sglang.test.test_utils import DEFAULT_URL_FOR_TEST, _parse_int_list_env
|
||||
|
||||
DEEPSEEK_V32_MODEL_PATH = "deepseek-ai/DeepSeek-V3.2-Exp"
|
||||
PROFILE_DIR = "performance_profiles_deepseek_v32"
|
||||
|
||||
|
||||
class TestNightlyDeepseekV32Basic(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.model = DEEPSEEK_V32_MODEL_PATH
|
||||
cls.base_url = DEFAULT_URL_FOR_TEST
|
||||
cls.batch_sizes = [1, 1, 8, 16, 64]
|
||||
cls.input_lens = tuple(_parse_int_list_env("NIGHTLY_INPUT_LENS", "4096"))
|
||||
cls.output_lens = tuple(_parse_int_list_env("NIGHTLY_OUTPUT_LENS", "512"))
|
||||
cls.other_args = [
|
||||
"--trust-remote-code",
|
||||
"--tp",
|
||||
"8",
|
||||
"--dp",
|
||||
"8",
|
||||
"--enable-dp-attention",
|
||||
]
|
||||
cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url)
|
||||
cls.runner.setup_profile_directory()
|
||||
|
||||
def test_bench_one_batch(self):
|
||||
results, success = self.runner.run_benchmark_for_model(
|
||||
model_path=self.model,
|
||||
batch_sizes=self.batch_sizes,
|
||||
input_lens=self.input_lens,
|
||||
output_lens=self.output_lens,
|
||||
other_args=self.other_args,
|
||||
variant="basic",
|
||||
)
|
||||
|
||||
self.runner.add_report(results)
|
||||
self.runner.write_final_report()
|
||||
|
||||
if not success:
|
||||
raise AssertionError(
|
||||
f"Benchmark failed for {self.model} with basic configuration"
|
||||
)
|
||||
|
||||
|
||||
class TestNightlyDeepseekV32MTP(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.model = DEEPSEEK_V32_MODEL_PATH
|
||||
cls.base_url = DEFAULT_URL_FOR_TEST
|
||||
cls.batch_sizes = [1, 1, 8, 16, 64]
|
||||
cls.input_lens = tuple(_parse_int_list_env("NIGHTLY_INPUT_LENS", "4096"))
|
||||
cls.output_lens = tuple(_parse_int_list_env("NIGHTLY_OUTPUT_LENS", "512"))
|
||||
cls.other_args = [
|
||||
"--trust-remote-code",
|
||||
"--tp",
|
||||
"8",
|
||||
"--dp",
|
||||
"8",
|
||||
"--enable-dp-attention",
|
||||
"--speculative-algorithm",
|
||||
"EAGLE",
|
||||
"--speculative-num-steps",
|
||||
"3",
|
||||
"--speculative-eagle-topk",
|
||||
"1",
|
||||
"--speculative-num-draft-tokens",
|
||||
"4",
|
||||
"--mem-frac",
|
||||
"0.7",
|
||||
]
|
||||
cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url)
|
||||
cls.runner.setup_profile_directory()
|
||||
|
||||
def test_bench_one_batch(self):
|
||||
results, success = self.runner.run_benchmark_for_model(
|
||||
model_path=self.model,
|
||||
batch_sizes=self.batch_sizes,
|
||||
input_lens=self.input_lens,
|
||||
output_lens=self.output_lens,
|
||||
other_args=self.other_args,
|
||||
variant="mtp",
|
||||
)
|
||||
|
||||
self.runner.add_report(results)
|
||||
self.runner.write_final_report()
|
||||
|
||||
if not success:
|
||||
raise AssertionError(
|
||||
f"Benchmark failed for {self.model} with MTP configuration"
|
||||
)
|
||||
|
||||
|
||||
class TestNightlyDeepseekV32NSA(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.model = DEEPSEEK_V32_MODEL_PATH
|
||||
cls.base_url = DEFAULT_URL_FOR_TEST
|
||||
cls.batch_sizes = [1, 1, 8, 16, 64]
|
||||
cls.input_lens = tuple(_parse_int_list_env("NIGHTLY_INPUT_LENS", "4096"))
|
||||
cls.output_lens = tuple(_parse_int_list_env("NIGHTLY_OUTPUT_LENS", "512"))
|
||||
cls.other_args = [
|
||||
"--trust-remote-code",
|
||||
"--tp",
|
||||
"8",
|
||||
"--dp",
|
||||
"8",
|
||||
"--enable-dp-attention",
|
||||
"--attention-backend",
|
||||
"nsa",
|
||||
"--nsa-prefill-backend",
|
||||
"flashmla_sparse",
|
||||
"--nsa-decode-backend",
|
||||
"flashmla_kv",
|
||||
]
|
||||
cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url)
|
||||
cls.runner.setup_profile_directory()
|
||||
|
||||
def test_bench_one_batch(self):
|
||||
results, success = self.runner.run_benchmark_for_model(
|
||||
model_path=self.model,
|
||||
batch_sizes=self.batch_sizes,
|
||||
input_lens=self.input_lens,
|
||||
output_lens=self.output_lens,
|
||||
other_args=self.other_args,
|
||||
variant="nsa",
|
||||
)
|
||||
|
||||
self.runner.add_report(results)
|
||||
self.runner.write_final_report()
|
||||
|
||||
if not success:
|
||||
raise AssertionError(
|
||||
f"Benchmark failed for {self.model} with NSA configuration"
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,58 @@
|
||||
import unittest
|
||||
|
||||
from nightly_utils import NightlyBenchmarkRunner
|
||||
|
||||
from sglang.test.test_utils import DEFAULT_URL_FOR_TEST
|
||||
|
||||
PROFILE_DIR = "performance_profiles_gpt_oss_4gpu"
|
||||
|
||||
|
||||
class TestNightlyGptOss4GpuPerformance(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.models = [
|
||||
(
|
||||
"openai/gpt-oss-120b",
|
||||
[
|
||||
"--tp",
|
||||
"4",
|
||||
"--cuda-graph-max-bs",
|
||||
"200",
|
||||
"--mem-fraction-static",
|
||||
"0.93",
|
||||
],
|
||||
),
|
||||
]
|
||||
cls.base_url = DEFAULT_URL_FOR_TEST
|
||||
cls.batch_sizes = [1, 1, 8, 16, 64]
|
||||
cls.input_lens = (4096,)
|
||||
cls.output_lens = (512,)
|
||||
cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url)
|
||||
cls.runner.setup_profile_directory()
|
||||
|
||||
def test_bench_one_batch(self):
|
||||
all_model_succeed = True
|
||||
|
||||
for model_path, other_args in self.models:
|
||||
with self.subTest(model=model_path):
|
||||
results, success = self.runner.run_benchmark_for_model(
|
||||
model_path=model_path,
|
||||
batch_sizes=self.batch_sizes,
|
||||
input_lens=self.input_lens,
|
||||
output_lens=self.output_lens,
|
||||
other_args=other_args,
|
||||
)
|
||||
|
||||
if not success:
|
||||
all_model_succeed = False
|
||||
|
||||
self.runner.add_report(results)
|
||||
|
||||
self.runner.write_final_report()
|
||||
|
||||
if not all_model_succeed:
|
||||
raise AssertionError("Some models failed the perf tests.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,194 @@
|
||||
import json
|
||||
import os
|
||||
import unittest
|
||||
import warnings
|
||||
from datetime import datetime
|
||||
from types import SimpleNamespace
|
||||
|
||||
from sglang.srt.utils import kill_process_tree
|
||||
from sglang.test.run_eval import run_eval
|
||||
from sglang.test.test_utils import (
|
||||
DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP1,
|
||||
DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP2,
|
||||
DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP1,
|
||||
DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP2,
|
||||
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||
DEFAULT_URL_FOR_TEST,
|
||||
is_in_ci,
|
||||
parse_models,
|
||||
popen_launch_server,
|
||||
write_github_step_summary,
|
||||
write_results_to_json,
|
||||
)
|
||||
|
||||
MODEL_SCORE_THRESHOLDS = {
|
||||
"meta-llama/Llama-3.1-8B-Instruct": 0.82,
|
||||
"mistralai/Mistral-7B-Instruct-v0.3": 0.58,
|
||||
"deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct": 0.85,
|
||||
"meta-llama/Llama-3.1-70B-Instruct": 0.95,
|
||||
"mistralai/Mixtral-8x7B-Instruct-v0.1": 0.64,
|
||||
"Qwen/Qwen2-57B-A14B-Instruct": 0.86,
|
||||
"neuralmagic/Meta-Llama-3.1-8B-Instruct-FP8": 0.83,
|
||||
"neuralmagic/Mistral-7B-Instruct-v0.3-FP8": 0.54,
|
||||
"neuralmagic/Meta-Llama-3.1-70B-Instruct-FP8": 0.94,
|
||||
"neuralmagic/Qwen2-72B-Instruct-FP8": 0.94,
|
||||
"neuralmagic/Qwen2-57B-A14B-Instruct-FP8": 0.86,
|
||||
"neuralmagic/Mixtral-8x7B-Instruct-v0.1-FP8": 0.65,
|
||||
"google/gemma-2-27b-it": 0.91,
|
||||
"neuralmagic/DeepSeek-Coder-V2-Lite-Instruct-FP8": 0.84,
|
||||
}
|
||||
|
||||
failing_models = {
|
||||
"neuralmagic/gemma-2-2b-it-FP8",
|
||||
}
|
||||
|
||||
|
||||
def remove_failing_models(model_str):
|
||||
models = model_str.split(",")
|
||||
filtered = [m for m in models if m not in failing_models]
|
||||
return ",".join(filtered)
|
||||
|
||||
|
||||
DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP1 = remove_failing_models(
|
||||
DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP1
|
||||
)
|
||||
DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP2 = remove_failing_models(
|
||||
DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP2
|
||||
)
|
||||
DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP1 = remove_failing_models(
|
||||
DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP1
|
||||
)
|
||||
DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP2 = remove_failing_models(
|
||||
DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP2
|
||||
)
|
||||
|
||||
NO_MOE_PADDING_MODELS = {"neuralmagic/Mixtral-8x7B-Instruct-v0.1-FP8"}
|
||||
DISABLE_HF_XET_MODELS = {
|
||||
"Qwen/Qwen2-57B-A14B-Instruct",
|
||||
"neuralmagic/Qwen2-57B-A14B-Instruct-FP8",
|
||||
}
|
||||
TRITON_MOE_MODELS = {
|
||||
"neuralmagic/Mixtral-8x7B-Instruct-v0.1-FP8",
|
||||
"neuralmagic/DeepSeek-Coder-V2-Lite-Instruct-FP8",
|
||||
"mistralai/Mixtral-8x7B-Instruct-v0.1",
|
||||
"mistralai/Mistral-7B-Instruct-v0.3",
|
||||
}
|
||||
|
||||
|
||||
def popen_launch_server_wrapper(base_url, model, is_tp2):
|
||||
other_args = ["--log-level-http", "warning", "--trust-remote-code"]
|
||||
if is_tp2:
|
||||
other_args.extend(["--tp", "2"])
|
||||
|
||||
process = popen_launch_server(
|
||||
model,
|
||||
base_url,
|
||||
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||
other_args=other_args,
|
||||
)
|
||||
return process
|
||||
|
||||
|
||||
def check_model_scores(results):
|
||||
failed_models = []
|
||||
summary = " | model | score | threshold |\n"
|
||||
summary += "| ----- | ----- | --------- |\n"
|
||||
|
||||
for model, score in results:
|
||||
threshold = MODEL_SCORE_THRESHOLDS.get(model)
|
||||
if threshold is None:
|
||||
print(f"Warning: No threshold defined for model {model}")
|
||||
continue
|
||||
|
||||
if score < threshold:
|
||||
failed_models.append(
|
||||
f"\nScore Check Failed: {model}\n"
|
||||
f"Model {model} score ({score:.4f}) is below threshold ({threshold:.4f})"
|
||||
)
|
||||
|
||||
line = f"| {model} | {score} | {threshold} |\n"
|
||||
summary += line
|
||||
|
||||
print(summary)
|
||||
|
||||
if is_in_ci():
|
||||
write_github_step_summary(f"### TestNightlyGsm8KEval\n{summary}")
|
||||
|
||||
if failed_models:
|
||||
raise AssertionError("\n".join(failed_models))
|
||||
|
||||
|
||||
# Do not use `CustomTestCase` since `test_mgsm_en_all_models` does not want retry
|
||||
class TestNightlyGsm8KEval(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.model_groups = [
|
||||
(parse_models(DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP1), False, False),
|
||||
(parse_models(DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP2), False, True),
|
||||
(parse_models(DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP1), True, False),
|
||||
(parse_models(DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP2), True, True),
|
||||
]
|
||||
cls.base_url = DEFAULT_URL_FOR_TEST
|
||||
|
||||
def test_mgsm_en_all_models(self):
|
||||
warnings.filterwarnings(
|
||||
"ignore", category=ResourceWarning, message="unclosed.*socket"
|
||||
)
|
||||
is_first = True
|
||||
all_results = []
|
||||
|
||||
for model_group, is_fp8, is_tp2 in self.model_groups:
|
||||
for model in model_group:
|
||||
with self.subTest(model=model):
|
||||
os.environ["SGLANG_MOE_PADDING"] = (
|
||||
"0" if model in NO_MOE_PADDING_MODELS else "1"
|
||||
)
|
||||
os.environ["HF_HUB_DISABLE_XET"] = (
|
||||
"1" if model in DISABLE_HF_XET_MODELS else "0"
|
||||
)
|
||||
os.environ["SGLANG_USE_AITER"] = (
|
||||
"0" if model in TRITON_MOE_MODELS else "1"
|
||||
)
|
||||
|
||||
process = popen_launch_server_wrapper(self.base_url, model, is_tp2)
|
||||
|
||||
args = SimpleNamespace(
|
||||
base_url=self.base_url,
|
||||
model=model,
|
||||
eval_name="mgsm_en",
|
||||
num_examples=None,
|
||||
num_threads=1024,
|
||||
)
|
||||
# Allow retries, so flaky errors are avoided.
|
||||
threshold = MODEL_SCORE_THRESHOLDS.get(model)
|
||||
for attempt in range(3):
|
||||
try:
|
||||
metrics = run_eval(args)
|
||||
score = metrics["score"]
|
||||
if score >= threshold:
|
||||
break
|
||||
except Exception as e:
|
||||
print(f"Attempt {attempt + 1} failed with error: {e}")
|
||||
print(
|
||||
f"{'=' * 42}\n{model} - metrics={metrics} score={metrics['score']}\n{'=' * 42}\n"
|
||||
)
|
||||
|
||||
write_results_to_json(model, metrics, "w" if is_first else "a")
|
||||
is_first = False
|
||||
|
||||
all_results.append((model, metrics["score"]))
|
||||
kill_process_tree(process.pid)
|
||||
|
||||
try:
|
||||
with open("results.json", "r") as f:
|
||||
print("\nFinal Results from results.json:")
|
||||
print(json.dumps(json.load(f), indent=2))
|
||||
except Exception as e:
|
||||
print(f"Error reading results.json: {e}")
|
||||
|
||||
# Check all scores after collecting all results
|
||||
check_model_scores(all_results)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,124 @@
|
||||
import json
|
||||
import unittest
|
||||
import warnings
|
||||
from types import SimpleNamespace
|
||||
|
||||
from sglang.srt.utils import kill_process_tree
|
||||
from sglang.test.run_eval import run_eval
|
||||
from sglang.test.test_utils import (
|
||||
DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP1,
|
||||
DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP2,
|
||||
DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP1,
|
||||
DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP2,
|
||||
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||
DEFAULT_URL_FOR_TEST,
|
||||
ModelLaunchSettings,
|
||||
check_evaluation_test_results,
|
||||
parse_models,
|
||||
popen_launch_server,
|
||||
write_results_to_json,
|
||||
)
|
||||
|
||||
MODEL_SCORE_THRESHOLDS = {
|
||||
"meta-llama/Llama-3.1-8B-Instruct": 0.82,
|
||||
"mistralai/Mistral-7B-Instruct-v0.3": 0.58,
|
||||
"deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct": 0.85,
|
||||
"google/gemma-2-27b-it": 0.91,
|
||||
"meta-llama/Llama-3.1-70B-Instruct": 0.95,
|
||||
"mistralai/Mixtral-8x7B-Instruct-v0.1": 0.616,
|
||||
"Qwen/Qwen2-57B-A14B-Instruct": 0.86,
|
||||
"neuralmagic/Meta-Llama-3.1-8B-Instruct-FP8": 0.83,
|
||||
"neuralmagic/Mistral-7B-Instruct-v0.3-FP8": 0.54,
|
||||
"neuralmagic/DeepSeek-Coder-V2-Lite-Instruct-FP8": 0.835,
|
||||
"zai-org/GLM-4.5-Air-FP8": 0.75,
|
||||
# The threshold of neuralmagic/gemma-2-2b-it-FP8 should be 0.6, but this model has some accuracy regression.
|
||||
# The fix is tracked at https://github.com/sgl-project/sglang/issues/4324, we set it to 0.50, for now, to make CI green.
|
||||
"neuralmagic/gemma-2-2b-it-FP8": 0.50,
|
||||
"neuralmagic/Meta-Llama-3.1-70B-Instruct-FP8": 0.94,
|
||||
"neuralmagic/Mixtral-8x7B-Instruct-v0.1-FP8": 0.65,
|
||||
"neuralmagic/Qwen2-72B-Instruct-FP8": 0.94,
|
||||
"neuralmagic/Qwen2-57B-A14B-Instruct-FP8": 0.82,
|
||||
}
|
||||
|
||||
|
||||
# Do not use `CustomTestCase` since `test_mgsm_en_all_models` does not want retry
|
||||
class TestNightlyGsm8KEval(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.models = []
|
||||
models_tp1 = parse_models(
|
||||
DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP1
|
||||
) + parse_models(DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP1)
|
||||
for model_path in models_tp1:
|
||||
cls.models.append(ModelLaunchSettings(model_path, tp_size=1))
|
||||
|
||||
models_tp2 = parse_models(
|
||||
DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP2
|
||||
) + parse_models(DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP2)
|
||||
for model_path in models_tp2:
|
||||
cls.models.append(ModelLaunchSettings(model_path, tp_size=2))
|
||||
|
||||
cls.base_url = DEFAULT_URL_FOR_TEST
|
||||
|
||||
def test_mgsm_en_all_models(self):
|
||||
warnings.filterwarnings(
|
||||
"ignore", category=ResourceWarning, message="unclosed.*socket"
|
||||
)
|
||||
is_first = True
|
||||
all_results = []
|
||||
for model_setup in self.models:
|
||||
with self.subTest(model=model_setup.model_path):
|
||||
other_args = list(model_setup.extra_args)
|
||||
|
||||
if model_setup.model_path == "meta-llama/Llama-3.1-70B-Instruct":
|
||||
other_args.extend(["--mem-fraction-static", "0.9"])
|
||||
|
||||
process = popen_launch_server(
|
||||
model=model_setup.model_path,
|
||||
other_args=other_args,
|
||||
base_url=self.base_url,
|
||||
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||
)
|
||||
|
||||
try:
|
||||
args = SimpleNamespace(
|
||||
base_url=self.base_url,
|
||||
model=model_setup.model_path,
|
||||
eval_name="mgsm_en",
|
||||
num_examples=None,
|
||||
num_threads=1024,
|
||||
)
|
||||
|
||||
metrics = run_eval(args)
|
||||
print(
|
||||
f"{'=' * 42}\n{model_setup.model_path} - metrics={metrics} score={metrics['score']}\n{'=' * 42}\n"
|
||||
)
|
||||
|
||||
write_results_to_json(
|
||||
model_setup.model_path, metrics, "w" if is_first else "a"
|
||||
)
|
||||
is_first = False
|
||||
|
||||
# 0.0 for empty latency
|
||||
all_results.append((model_setup.model_path, metrics["score"], 0.0))
|
||||
finally:
|
||||
kill_process_tree(process.pid)
|
||||
|
||||
try:
|
||||
with open("results.json", "r") as f:
|
||||
print("\nFinal Results from results.json:")
|
||||
print(json.dumps(json.load(f), indent=2))
|
||||
except Exception as e:
|
||||
print(f"Error reading results.json: {e}")
|
||||
|
||||
# Check all scores after collecting all results
|
||||
check_evaluation_test_results(
|
||||
all_results,
|
||||
self.__class__.__name__,
|
||||
model_accuracy_thresholds=MODEL_SCORE_THRESHOLDS,
|
||||
model_count=len(self.models),
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,60 @@
|
||||
import unittest
|
||||
|
||||
from nightly_utils import NightlyBenchmarkRunner
|
||||
|
||||
from sglang.test.test_utils import (
|
||||
DEFAULT_URL_FOR_TEST,
|
||||
ModelLaunchSettings,
|
||||
_parse_int_list_env,
|
||||
parse_models,
|
||||
)
|
||||
|
||||
PROFILE_DIR = "performance_profiles_text_models"
|
||||
|
||||
|
||||
class TestNightlyTextModelsPerformance(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.models = []
|
||||
# TODO: replace with DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP1 or other model lists
|
||||
for model_path in parse_models("meta-llama/Llama-3.1-8B-Instruct"):
|
||||
cls.models.append(ModelLaunchSettings(model_path, tp_size=1))
|
||||
for model_path in parse_models("Qwen/Qwen2-57B-A14B-Instruct"):
|
||||
cls.models.append(ModelLaunchSettings(model_path, tp_size=2))
|
||||
# (parse_models(DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP1), False, False),
|
||||
# (parse_models(DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP2), False, True),
|
||||
# (parse_models(DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP1), True, False),
|
||||
# (parse_models(DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP2), True, True),
|
||||
cls.base_url = DEFAULT_URL_FOR_TEST
|
||||
cls.batch_sizes = [1, 1, 8, 16, 64]
|
||||
cls.input_lens = tuple(_parse_int_list_env("NIGHTLY_INPUT_LENS", "4096"))
|
||||
cls.output_lens = tuple(_parse_int_list_env("NIGHTLY_OUTPUT_LENS", "512"))
|
||||
cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url)
|
||||
cls.runner.setup_profile_directory()
|
||||
|
||||
def test_bench_one_batch(self):
|
||||
all_model_succeed = True
|
||||
|
||||
for model_setup in self.models:
|
||||
with self.subTest(model=model_setup.model_path):
|
||||
results, success = self.runner.run_benchmark_for_model(
|
||||
model_path=model_setup.model_path,
|
||||
batch_sizes=self.batch_sizes,
|
||||
input_lens=self.input_lens,
|
||||
output_lens=self.output_lens,
|
||||
other_args=model_setup.extra_args,
|
||||
)
|
||||
|
||||
if not success:
|
||||
all_model_succeed = False
|
||||
|
||||
self.runner.add_report(results)
|
||||
|
||||
self.runner.write_final_report()
|
||||
|
||||
if not all_model_succeed:
|
||||
raise AssertionError("Some models failed the perf tests.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,127 @@
|
||||
import json
|
||||
import unittest
|
||||
import warnings
|
||||
from types import SimpleNamespace
|
||||
|
||||
from sglang.srt.utils import kill_process_tree
|
||||
from sglang.test.run_eval import run_eval
|
||||
from sglang.test.test_utils import (
|
||||
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||
DEFAULT_URL_FOR_TEST,
|
||||
ModelEvalMetrics,
|
||||
ModelLaunchSettings,
|
||||
check_evaluation_test_results,
|
||||
popen_launch_server,
|
||||
write_results_to_json,
|
||||
)
|
||||
|
||||
MODEL_THRESHOLDS = {
|
||||
# Conservative thresholds on 100 MMMU samples, especially for latency thresholds
|
||||
ModelLaunchSettings("deepseek-ai/deepseek-vl2-small"): ModelEvalMetrics(
|
||||
0.330, 56.1
|
||||
),
|
||||
ModelLaunchSettings("deepseek-ai/Janus-Pro-7B"): ModelEvalMetrics(0.285, 40.3),
|
||||
ModelLaunchSettings("Efficient-Large-Model/NVILA-8B-hf"): ModelEvalMetrics(
|
||||
0.270, 56.7
|
||||
),
|
||||
ModelLaunchSettings("Efficient-Large-Model/NVILA-Lite-2B-hf"): ModelEvalMetrics(
|
||||
0.270, 23.8
|
||||
),
|
||||
ModelLaunchSettings("google/gemma-3-4b-it"): ModelEvalMetrics(0.360, 10.9),
|
||||
ModelLaunchSettings("google/gemma-3n-E4B-it"): ModelEvalMetrics(0.360, 17.7),
|
||||
ModelLaunchSettings("mistral-community/pixtral-12b"): ModelEvalMetrics(0.360, 16.6),
|
||||
ModelLaunchSettings("moonshotai/Kimi-VL-A3B-Instruct"): ModelEvalMetrics(
|
||||
0.330, 22.3
|
||||
),
|
||||
ModelLaunchSettings("openbmb/MiniCPM-o-2_6"): ModelEvalMetrics(0.330, 29.3),
|
||||
ModelLaunchSettings("openbmb/MiniCPM-v-2_6"): ModelEvalMetrics(0.259, 36.3),
|
||||
ModelLaunchSettings("OpenGVLab/InternVL2_5-2B"): ModelEvalMetrics(0.300, 17.0),
|
||||
ModelLaunchSettings("Qwen/Qwen2-VL-7B-Instruct"): ModelEvalMetrics(0.310, 83.3),
|
||||
ModelLaunchSettings("Qwen/Qwen2.5-VL-7B-Instruct"): ModelEvalMetrics(0.340, 31.9),
|
||||
ModelLaunchSettings(
|
||||
"Qwen/Qwen3-VL-30B-A3B-Instruct", extra_args=["--tp=2"]
|
||||
): ModelEvalMetrics(0.29, 37.0),
|
||||
ModelLaunchSettings(
|
||||
"unsloth/Mistral-Small-3.1-24B-Instruct-2503"
|
||||
): ModelEvalMetrics(0.310, 16.7),
|
||||
ModelLaunchSettings("XiaomiMiMo/MiMo-VL-7B-RL"): ModelEvalMetrics(0.28, 32.0),
|
||||
ModelLaunchSettings("zai-org/GLM-4.1V-9B-Thinking"): ModelEvalMetrics(0.280, 30.4),
|
||||
}
|
||||
|
||||
|
||||
class TestNightlyVLMMmmuEval(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.models = list(MODEL_THRESHOLDS.keys())
|
||||
cls.base_url = DEFAULT_URL_FOR_TEST
|
||||
|
||||
def test_mmmu_vlm_models(self):
|
||||
warnings.filterwarnings(
|
||||
"ignore", category=ResourceWarning, message="unclosed.*socket"
|
||||
)
|
||||
is_first = True
|
||||
all_results = []
|
||||
|
||||
for model in self.models:
|
||||
model_path = model.model_path
|
||||
with self.subTest(model=model_path):
|
||||
process = popen_launch_server(
|
||||
model=model_path,
|
||||
base_url=self.base_url,
|
||||
other_args=model.extra_args,
|
||||
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||
)
|
||||
try:
|
||||
args = SimpleNamespace(
|
||||
base_url=self.base_url,
|
||||
model=model_path,
|
||||
eval_name="mmmu",
|
||||
num_examples=100,
|
||||
num_threads=64,
|
||||
max_tokens=30,
|
||||
)
|
||||
|
||||
args.return_latency = True
|
||||
|
||||
metrics, latency = run_eval(args)
|
||||
|
||||
metrics["score"] = round(metrics["score"], 4)
|
||||
metrics["latency"] = round(latency, 4)
|
||||
print(
|
||||
f"{'=' * 42}\n{model_path} - metrics={metrics} score={metrics['score']}\n{'=' * 42}\n"
|
||||
)
|
||||
|
||||
write_results_to_json(model_path, metrics, "w" if is_first else "a")
|
||||
is_first = False
|
||||
|
||||
all_results.append(
|
||||
(model_path, metrics["score"], metrics["latency"])
|
||||
)
|
||||
finally:
|
||||
kill_process_tree(process.pid)
|
||||
|
||||
try:
|
||||
with open("results.json", "r") as f:
|
||||
print("\nFinal Results from results.json:")
|
||||
print(json.dumps(json.load(f), indent=2))
|
||||
except Exception as e:
|
||||
print(f"Error reading results: {e}")
|
||||
|
||||
model_accuracy_thresholds = {
|
||||
model.model_path: threshold.accuracy
|
||||
for model, threshold in MODEL_THRESHOLDS.items()
|
||||
}
|
||||
model_latency_thresholds = {
|
||||
model.model_path: threshold.eval_time
|
||||
for model, threshold in MODEL_THRESHOLDS.items()
|
||||
}
|
||||
check_evaluation_test_results(
|
||||
all_results,
|
||||
self.__class__.__name__,
|
||||
model_accuracy_thresholds=model_accuracy_thresholds,
|
||||
model_latency_thresholds=model_latency_thresholds,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,88 @@
|
||||
import os
|
||||
import unittest
|
||||
import warnings
|
||||
|
||||
from nightly_utils import NightlyBenchmarkRunner
|
||||
|
||||
from sglang.test.test_utils import (
|
||||
DEFAULT_URL_FOR_TEST,
|
||||
ModelLaunchSettings,
|
||||
_parse_int_list_env,
|
||||
parse_models,
|
||||
)
|
||||
|
||||
PROFILE_DIR = "performance_profiles_vlms"
|
||||
|
||||
MODEL_DEFAULTS = [
|
||||
# Keep conservative defaults. Can be overridden by env NIGHTLY_VLM_MODELS
|
||||
ModelLaunchSettings(
|
||||
"Qwen/Qwen2.5-VL-7B-Instruct",
|
||||
extra_args=["--mem-fraction-static=0.7"],
|
||||
),
|
||||
ModelLaunchSettings(
|
||||
"google/gemma-3-27b-it",
|
||||
),
|
||||
ModelLaunchSettings("Qwen/Qwen3-VL-30B-A3B-Instruct", extra_args=["--tp=2"]),
|
||||
# "OpenGVLab/InternVL2_5-2B",
|
||||
# buggy in official transformers impl
|
||||
# "openbmb/MiniCPM-V-2_6",
|
||||
]
|
||||
|
||||
|
||||
class TestNightlyVLMModelsPerformance(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
warnings.filterwarnings(
|
||||
"ignore", category=ResourceWarning, message="unclosed.*socket"
|
||||
)
|
||||
|
||||
nightly_vlm_models_str = os.environ.get("NIGHTLY_VLM_MODELS")
|
||||
if nightly_vlm_models_str:
|
||||
cls.models = []
|
||||
model_paths = parse_models(nightly_vlm_models_str)
|
||||
for model_path in model_paths:
|
||||
cls.models.append(ModelLaunchSettings(model_path))
|
||||
else:
|
||||
cls.models = MODEL_DEFAULTS
|
||||
|
||||
cls.base_url = DEFAULT_URL_FOR_TEST
|
||||
|
||||
cls.batch_sizes = _parse_int_list_env("NIGHTLY_VLM_BATCH_SIZES", "1,1,2,8,16")
|
||||
cls.input_lens = tuple(_parse_int_list_env("NIGHTLY_VLM_INPUT_LENS", "4096"))
|
||||
cls.output_lens = tuple(_parse_int_list_env("NIGHTLY_VLM_OUTPUT_LENS", "512"))
|
||||
cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url)
|
||||
cls.runner.setup_profile_directory()
|
||||
|
||||
def test_bench_one_batch(self):
|
||||
all_model_succeed = True
|
||||
|
||||
for model_setup in self.models:
|
||||
with self.subTest(model=model_setup.model_path):
|
||||
# VLMs need additional benchmark args for dataset and trust-remote-code
|
||||
extra_bench_args = [
|
||||
"--trust-remote-code",
|
||||
"--dataset-name=mmmu",
|
||||
]
|
||||
|
||||
results, success = self.runner.run_benchmark_for_model(
|
||||
model_path=model_setup.model_path,
|
||||
batch_sizes=self.batch_sizes,
|
||||
input_lens=self.input_lens,
|
||||
output_lens=self.output_lens,
|
||||
other_args=model_setup.extra_args,
|
||||
extra_bench_args=extra_bench_args,
|
||||
)
|
||||
|
||||
if not success:
|
||||
all_model_succeed = False
|
||||
|
||||
self.runner.add_report(results)
|
||||
|
||||
self.runner.write_final_report()
|
||||
|
||||
if not all_model_succeed:
|
||||
raise AssertionError("Some models failed the perf tests.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user