[CI] Move nightly tests to test/nightly/ (#13683)
This commit is contained in:
alisonshao
@@ -1,305 +0,0 @@
|
||||
"""Utilities for running nightly performance benchmarks with profiling."""
|
||||
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import time
|
||||
from typing import List, Optional, Tuple
|
||||
|
||||
from sglang.bench_one_batch_server import BenchmarkResult, generate_markdown_report
|
||||
from sglang.srt.utils import kill_process_tree
|
||||
from sglang.test.test_utils import (
|
||||
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||
is_in_ci,
|
||||
popen_launch_server,
|
||||
write_github_step_summary,
|
||||
)
|
||||
|
||||
|
||||
class NightlyBenchmarkRunner:
|
||||
"""Helper class for running nightly performance benchmarks with profiling.
|
||||
|
||||
This class encapsulates common patterns used across nightly performance tests,
|
||||
including profile directory management, benchmark command construction,
|
||||
result parsing, and report generation.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
profile_dir: str,
|
||||
test_name: str,
|
||||
base_url: str,
|
||||
gpu_config: str = None,
|
||||
):
|
||||
"""Initialize the benchmark runner.
|
||||
|
||||
Args:
|
||||
profile_dir: Directory to store performance profiles
|
||||
test_name: Name of the test (used for reporting)
|
||||
base_url: Base URL for the server
|
||||
gpu_config: Optional GPU configuration string (e.g., "2-gpu-h100", "8-gpu-b200")
|
||||
"""
|
||||
self.profile_dir = profile_dir
|
||||
self.test_name = test_name
|
||||
self.base_url = base_url
|
||||
self.gpu_config = gpu_config or os.environ.get("GPU_CONFIG", "")
|
||||
|
||||
# Include GPU config in report header if available
|
||||
header = f"## {test_name}"
|
||||
if self.gpu_config:
|
||||
header += f" ({self.gpu_config})"
|
||||
header += "\n"
|
||||
self.full_report = header + BenchmarkResult.help_str()
|
||||
|
||||
def setup_profile_directory(self) -> None:
|
||||
"""Create the profile directory if it doesn't exist."""
|
||||
os.makedirs(self.profile_dir, exist_ok=True)
|
||||
|
||||
def generate_profile_filename(
|
||||
self, model_path: str, variant: str = ""
|
||||
) -> Tuple[str, str]:
|
||||
"""Generate unique profile filename and path for the model.
|
||||
|
||||
Args:
|
||||
model_path: Path to the model (e.g., "deepseek-ai/DeepSeek-V3.1")
|
||||
variant: Optional variant suffix (e.g., "basic", "mtp", "nsa")
|
||||
|
||||
Returns:
|
||||
Tuple of (profile_path_prefix, json_output_file)
|
||||
"""
|
||||
timestamp = int(time.time())
|
||||
model_safe_name = model_path.replace("/", "_")
|
||||
|
||||
# Build filename with optional variant
|
||||
if variant:
|
||||
profile_filename = f"{model_safe_name}_{variant}_{timestamp}"
|
||||
json_filename = f"results_{model_safe_name}_{variant}_{timestamp}.json"
|
||||
else:
|
||||
profile_filename = f"{model_safe_name}_{timestamp}"
|
||||
json_filename = f"results_{model_safe_name}_{timestamp}.json"
|
||||
|
||||
profile_path_prefix = os.path.join(self.profile_dir, profile_filename)
|
||||
|
||||
return profile_path_prefix, json_filename
|
||||
|
||||
def build_benchmark_command(
|
||||
self,
|
||||
model_path: str,
|
||||
batch_sizes: List[int],
|
||||
input_lens: Tuple[int, ...],
|
||||
output_lens: Tuple[int, ...],
|
||||
profile_path_prefix: str,
|
||||
json_output_file: str,
|
||||
extra_args: Optional[List[str]] = None,
|
||||
) -> List[str]:
|
||||
"""Build the benchmark command with all required arguments.
|
||||
|
||||
Args:
|
||||
model_path: Path to the model
|
||||
batch_sizes: List of batch sizes to test
|
||||
input_lens: Tuple of input lengths to test
|
||||
output_lens: Tuple of output lengths to test
|
||||
profile_path_prefix: Prefix for profile output files
|
||||
json_output_file: Path to JSON output file
|
||||
extra_args: Optional extra arguments to append to command
|
||||
|
||||
Returns:
|
||||
List of command arguments ready for subprocess.run()
|
||||
"""
|
||||
command = [
|
||||
"python3",
|
||||
"-m",
|
||||
"sglang.bench_one_batch_server",
|
||||
"--model",
|
||||
model_path,
|
||||
"--base-url",
|
||||
self.base_url,
|
||||
"--batch-size",
|
||||
*[str(x) for x in batch_sizes],
|
||||
"--input-len",
|
||||
*[str(x) for x in input_lens],
|
||||
"--output-len",
|
||||
*[str(x) for x in output_lens],
|
||||
"--show-report",
|
||||
"--profile",
|
||||
"--profile-by-stage",
|
||||
"--profile-filename-prefix",
|
||||
profile_path_prefix,
|
||||
f"--output-path={json_output_file}",
|
||||
"--no-append-to-github-summary",
|
||||
]
|
||||
|
||||
if extra_args:
|
||||
command.extend(extra_args)
|
||||
|
||||
return command
|
||||
|
||||
def run_benchmark_command(
|
||||
self, command: List[str], model_description: str = ""
|
||||
) -> Tuple[subprocess.CompletedProcess, bool]:
|
||||
"""Execute the benchmark command and return the result.
|
||||
|
||||
Args:
|
||||
command: Command to execute
|
||||
model_description: Description for logging (e.g., "model_name (variant)")
|
||||
|
||||
Returns:
|
||||
Tuple of (CompletedProcess, success_bool)
|
||||
"""
|
||||
print(f"Running command: {' '.join(command)}")
|
||||
result = subprocess.run(command, capture_output=True, text=True)
|
||||
|
||||
if result.returncode != 0:
|
||||
desc = model_description or "benchmark"
|
||||
print(f"Error running benchmark for {desc}:")
|
||||
print(result.stderr)
|
||||
return result, False
|
||||
|
||||
return result, True
|
||||
|
||||
def load_benchmark_results(
|
||||
self, json_output_file: str, model_description: str = ""
|
||||
) -> Tuple[List[BenchmarkResult], bool]:
|
||||
"""Load and parse benchmark results from JSON file.
|
||||
|
||||
Args:
|
||||
json_output_file: Path to JSON output file
|
||||
model_description: Description for logging
|
||||
|
||||
Returns:
|
||||
Tuple of (list of BenchmarkResult objects, success_bool)
|
||||
"""
|
||||
benchmark_results = []
|
||||
|
||||
if not os.path.exists(json_output_file):
|
||||
desc = model_description or "model"
|
||||
print(f"Warning: JSON output file {json_output_file} not found for {desc}")
|
||||
return benchmark_results, False
|
||||
|
||||
try:
|
||||
with open(json_output_file, "r") as f:
|
||||
json_data = json.load(f)
|
||||
|
||||
# Convert JSON data to BenchmarkResult objects
|
||||
for data in json_data:
|
||||
benchmark_result = BenchmarkResult(**data)
|
||||
benchmark_results.append(benchmark_result)
|
||||
|
||||
print(
|
||||
f"Loaded {len(benchmark_results)} benchmark results from {json_output_file}"
|
||||
)
|
||||
|
||||
# Clean up JSON file
|
||||
os.remove(json_output_file)
|
||||
|
||||
return benchmark_results, True
|
||||
|
||||
except Exception as e:
|
||||
desc = model_description or "model"
|
||||
print(f"Error loading benchmark results for {desc}: {e}")
|
||||
# Try to clean up the file anyway
|
||||
if os.path.exists(json_output_file):
|
||||
os.remove(json_output_file)
|
||||
return benchmark_results, False
|
||||
|
||||
def run_benchmark_for_model(
|
||||
self,
|
||||
model_path: str,
|
||||
batch_sizes: List[int],
|
||||
input_lens: Tuple[int, ...],
|
||||
output_lens: Tuple[int, ...],
|
||||
other_args: Optional[List[str]] = None,
|
||||
variant: str = "",
|
||||
extra_bench_args: Optional[List[str]] = None,
|
||||
) -> Tuple[List[BenchmarkResult], bool]:
|
||||
"""Run a complete benchmark for a single model with server management.
|
||||
|
||||
This method handles:
|
||||
- Server launch and cleanup
|
||||
- Profile filename generation
|
||||
- Benchmark command construction and execution
|
||||
- Result loading and parsing
|
||||
|
||||
Args:
|
||||
model_path: Path to the model
|
||||
batch_sizes: List of batch sizes to test
|
||||
input_lens: Tuple of input lengths
|
||||
output_lens: Tuple of output lengths
|
||||
other_args: Arguments to pass to server launch
|
||||
variant: Optional variant suffix (e.g., "basic", "mtp")
|
||||
extra_bench_args: Extra arguments for the benchmark command
|
||||
|
||||
Returns:
|
||||
Tuple of (list of BenchmarkResult objects, success_bool)
|
||||
"""
|
||||
benchmark_results = []
|
||||
model_description = f"{model_path}" + (f" ({variant})" if variant else "")
|
||||
|
||||
# Launch server
|
||||
process = popen_launch_server(
|
||||
model=model_path,
|
||||
base_url=self.base_url,
|
||||
other_args=other_args or [],
|
||||
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||
)
|
||||
|
||||
try:
|
||||
# Generate filenames
|
||||
profile_path_prefix, json_output_file = self.generate_profile_filename(
|
||||
model_path, variant
|
||||
)
|
||||
|
||||
# Build and run benchmark command
|
||||
# Prepare extra args with run_name if variant is specified
|
||||
bench_args = list(extra_bench_args) if extra_bench_args else []
|
||||
if variant:
|
||||
bench_args.extend(["--run-name", variant])
|
||||
|
||||
command = self.build_benchmark_command(
|
||||
model_path,
|
||||
batch_sizes,
|
||||
input_lens,
|
||||
output_lens,
|
||||
profile_path_prefix,
|
||||
json_output_file,
|
||||
extra_args=bench_args,
|
||||
)
|
||||
|
||||
result, cmd_success = self.run_benchmark_command(command, model_description)
|
||||
|
||||
if not cmd_success:
|
||||
return benchmark_results, False
|
||||
|
||||
# Load results
|
||||
benchmark_results, load_success = self.load_benchmark_results(
|
||||
json_output_file, model_description
|
||||
)
|
||||
|
||||
return benchmark_results, load_success
|
||||
|
||||
finally:
|
||||
# Always clean up server process
|
||||
kill_process_tree(process.pid)
|
||||
|
||||
def add_report(self, results: List[BenchmarkResult]) -> None:
|
||||
"""Add benchmark results to the full report.
|
||||
|
||||
Args:
|
||||
results: List of BenchmarkResult objects to add to report
|
||||
"""
|
||||
if results:
|
||||
report_part = generate_markdown_report(self.profile_dir, results)
|
||||
self.full_report += report_part + "\n"
|
||||
|
||||
def write_final_report(self) -> None:
|
||||
"""Write the final report to GitHub summary if in CI."""
|
||||
if is_in_ci():
|
||||
write_github_step_summary(self.full_report)
|
||||
|
||||
def get_full_report(self) -> str:
|
||||
"""Get the accumulated full report.
|
||||
|
||||
Returns:
|
||||
The full markdown report as a string
|
||||
"""
|
||||
return self.full_report
|
||||
@@ -1,270 +0,0 @@
|
||||
import argparse
|
||||
import glob
|
||||
import json
|
||||
import os
|
||||
import random
|
||||
import subprocess
|
||||
import sys
|
||||
import unittest
|
||||
from types import SimpleNamespace
|
||||
|
||||
from sglang.srt.utils import kill_process_tree
|
||||
from sglang.test.test_utils import (
|
||||
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||
DEFAULT_URL_FOR_TEST,
|
||||
CustomTestCase,
|
||||
is_in_ci,
|
||||
popen_launch_server,
|
||||
)
|
||||
|
||||
MODELS = [
|
||||
SimpleNamespace(model="Qwen/Qwen2.5-VL-72B-Instruct", mmmu_accuracy=0.55),
|
||||
]
|
||||
|
||||
|
||||
# Set default mem_fraction_static to 0.8
|
||||
DEFAULT_MEM_FRACTION_STATIC = 0.8
|
||||
|
||||
|
||||
class TestVLMEncoderDP(CustomTestCase):
|
||||
parsed_args = None # Class variable to store args
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
# Removed argument parsing from here
|
||||
cls.base_url = DEFAULT_URL_FOR_TEST
|
||||
cls.api_key = "sk-123456"
|
||||
cls.time_out = DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH
|
||||
|
||||
if cls.parsed_args is None:
|
||||
cls.parsed_args = SimpleNamespace(
|
||||
mem_fraction_static=DEFAULT_MEM_FRACTION_STATIC
|
||||
)
|
||||
|
||||
# Set OpenAI API key and base URL environment variables. Needed for lmm-evals to work.
|
||||
os.environ["OPENAI_API_KEY"] = cls.api_key
|
||||
os.environ["OPENAI_API_BASE"] = f"{cls.base_url}/v1"
|
||||
|
||||
def run_mmmu_eval(
|
||||
self,
|
||||
model_version: str,
|
||||
output_path: str,
|
||||
*,
|
||||
env: dict | None = None,
|
||||
):
|
||||
"""
|
||||
Evaluate a VLM on the MMMU validation set with lmms‑eval.
|
||||
Only `model_version` (checkpoint) and `chat_template` vary;
|
||||
We are focusing only on the validation set due to resource constraints.
|
||||
"""
|
||||
# -------- fixed settings --------
|
||||
model = "openai_compatible"
|
||||
tp = 1
|
||||
tasks = "mmmu_val"
|
||||
batch_size = 32
|
||||
log_suffix = "openai_compatible"
|
||||
os.makedirs(output_path, exist_ok=True)
|
||||
|
||||
# -------- compose --model_args --------
|
||||
model_args = f'model_version="{model_version}",' f"tp={tp}"
|
||||
|
||||
# -------- build command list --------
|
||||
cmd = [
|
||||
"python3",
|
||||
"-m",
|
||||
"lmms_eval",
|
||||
"--model",
|
||||
model,
|
||||
"--model_args",
|
||||
model_args,
|
||||
"--tasks",
|
||||
tasks,
|
||||
"--batch_size",
|
||||
str(batch_size),
|
||||
"--log_samples",
|
||||
"--log_samples_suffix",
|
||||
log_suffix,
|
||||
"--output_path",
|
||||
str(output_path),
|
||||
]
|
||||
|
||||
subprocess.run(
|
||||
cmd,
|
||||
check=True,
|
||||
timeout=3600,
|
||||
)
|
||||
|
||||
def _run_vlm_mmmu_test(
|
||||
self,
|
||||
model,
|
||||
output_path,
|
||||
test_name="",
|
||||
custom_env=None,
|
||||
log_level="info",
|
||||
capture_output=False,
|
||||
):
|
||||
"""
|
||||
Common method to run VLM MMMU benchmark test.
|
||||
|
||||
Args:
|
||||
model: Model to test
|
||||
output_path: Path for output logs
|
||||
test_name: Optional test name for logging
|
||||
custom_env: Optional custom environment variables
|
||||
log_level: Log level for server (default: "info")
|
||||
capture_output: Whether to capture server stdout/stderr
|
||||
"""
|
||||
print(f"\nTesting model: {model.model}{test_name}")
|
||||
|
||||
process = None
|
||||
mmmu_accuracy = 0 # Initialize to handle potential exceptions
|
||||
server_output = ""
|
||||
|
||||
try:
|
||||
# Prepare environment variables
|
||||
process_env = os.environ.copy()
|
||||
if custom_env:
|
||||
process_env.update(custom_env)
|
||||
# if test vlm with cuda_ipc feature, open this env_var
|
||||
process_env["SGLANG_USE_CUDA_IPC_TRANSPORT"] = "1"
|
||||
|
||||
# Prepare stdout/stderr redirection if needed
|
||||
stdout_file = None
|
||||
stderr_file = None
|
||||
if capture_output:
|
||||
stdout_file = open("/tmp/server_stdout.log", "w")
|
||||
stderr_file = open("/tmp/server_stderr.log", "w")
|
||||
|
||||
# Launch server for testing
|
||||
process = popen_launch_server(
|
||||
model.model,
|
||||
base_url=self.base_url,
|
||||
timeout=self.time_out,
|
||||
api_key=self.api_key,
|
||||
other_args=[
|
||||
"--trust-remote-code",
|
||||
"--cuda-graph-max-bs",
|
||||
"32",
|
||||
"--mm-enable-dp-encoder",
|
||||
"--tp=4",
|
||||
"--mem-fraction-static",
|
||||
str(self.parsed_args.mem_fraction_static), # Use class variable
|
||||
"--log-level",
|
||||
log_level,
|
||||
],
|
||||
env=process_env,
|
||||
return_stdout_stderr=(
|
||||
(stdout_file, stderr_file) if capture_output else None
|
||||
),
|
||||
)
|
||||
|
||||
# Run evaluation
|
||||
self.run_mmmu_eval(model.model, output_path)
|
||||
|
||||
# Get the result file
|
||||
# Search recursively for JSON result files (lmms-eval v0.4.1+ creates subdirectories)
|
||||
result_files = glob.glob(f"{output_path}/**/*.json", recursive=True)
|
||||
if not result_files:
|
||||
result_files = glob.glob(f"{output_path}/*.json")
|
||||
|
||||
if not result_files:
|
||||
raise FileNotFoundError(f"No JSON result files found in {output_path}")
|
||||
|
||||
result_file_path = result_files[0]
|
||||
|
||||
with open(result_file_path, "r") as f:
|
||||
result = json.load(f)
|
||||
print(f"Result{test_name}\n: {result}")
|
||||
|
||||
# Process the result
|
||||
mmmu_accuracy = result["results"]["mmmu_val"]["mmmu_acc,none"]
|
||||
print(
|
||||
f"Model {model.model} achieved accuracy{test_name}: {mmmu_accuracy:.4f}"
|
||||
)
|
||||
|
||||
# Capture server output if requested
|
||||
if capture_output and process:
|
||||
server_output = self._read_output_from_files()
|
||||
|
||||
# Assert performance meets expected threshold
|
||||
self.assertGreaterEqual(
|
||||
mmmu_accuracy,
|
||||
model.mmmu_accuracy,
|
||||
f"Model {model.model} accuracy ({mmmu_accuracy:.4f}) below expected threshold ({model.mmmu_accuracy:.4f}){test_name}",
|
||||
)
|
||||
|
||||
return server_output
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error testing {model.model}{test_name}: {e}")
|
||||
self.fail(f"Test failed for {model.model}{test_name}: {e}")
|
||||
|
||||
finally:
|
||||
# Ensure process cleanup happens regardless of success/failure
|
||||
if process is not None and process.poll() is None:
|
||||
print(f"Cleaning up process {process.pid}")
|
||||
try:
|
||||
kill_process_tree(process.pid)
|
||||
except Exception as e:
|
||||
print(f"Error killing process: {e}")
|
||||
|
||||
# clean up temporary files
|
||||
if capture_output:
|
||||
if stdout_file:
|
||||
stdout_file.close()
|
||||
if stderr_file:
|
||||
stderr_file.close()
|
||||
for filename in ["/tmp/server_stdout.log", "/tmp/server_stderr.log"]:
|
||||
try:
|
||||
if os.path.exists(filename):
|
||||
os.remove(filename)
|
||||
except Exception as e:
|
||||
print(f"Error removing {filename}: {e}")
|
||||
|
||||
def _read_output_from_files(self):
|
||||
output_lines = []
|
||||
|
||||
log_files = [
|
||||
("/tmp/server_stdout.log", "[STDOUT]"),
|
||||
("/tmp/server_stderr.log", "[STDERR]"),
|
||||
]
|
||||
for filename, tag in log_files:
|
||||
try:
|
||||
if os.path.exists(filename):
|
||||
with open(filename, "r") as f:
|
||||
for line in f:
|
||||
output_lines.append(f"{tag} {line.rstrip()}")
|
||||
except Exception as e:
|
||||
print(f"Error reading {tag.lower()} file: {e}")
|
||||
|
||||
return "\n".join(output_lines)
|
||||
|
||||
def test_vlm_mmmu_benchmark(self):
|
||||
"""Test VLM models against MMMU benchmark."""
|
||||
models_to_test = MODELS
|
||||
|
||||
if is_in_ci():
|
||||
models_to_test = [random.choice(MODELS)]
|
||||
|
||||
for model in models_to_test:
|
||||
self._run_vlm_mmmu_test(model, "./logs")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# Define and parse arguments here, before unittest.main
|
||||
parser = argparse.ArgumentParser(description="Test VLM models")
|
||||
parser.add_argument(
|
||||
"--mem-fraction-static",
|
||||
type=float,
|
||||
help="Static memory fraction for the model",
|
||||
default=DEFAULT_MEM_FRACTION_STATIC,
|
||||
)
|
||||
|
||||
# Parse args intended for unittest
|
||||
args = parser.parse_args()
|
||||
|
||||
# Store the parsed args object on the class
|
||||
TestVLMEncoderDP.parsed_args = args
|
||||
|
||||
# Pass args to unittest
|
||||
unittest.main(argv=[sys.argv[0]])
|
||||
@@ -1,62 +0,0 @@
|
||||
import os
|
||||
import unittest
|
||||
from types import SimpleNamespace
|
||||
|
||||
from sglang.srt.utils import kill_process_tree
|
||||
from sglang.test.few_shot_gsm8k import run_eval
|
||||
from sglang.test.test_utils import (
|
||||
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||
DEFAULT_URL_FOR_TEST,
|
||||
CustomTestCase,
|
||||
popen_launch_server,
|
||||
)
|
||||
|
||||
|
||||
class TestFlashinferTrtllmGenAttnBackend(CustomTestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.model = "Qwen/Qwen3-Next-80B-A3B-Instruct"
|
||||
cls.base_url = DEFAULT_URL_FOR_TEST
|
||||
cls.process = popen_launch_server(
|
||||
cls.model,
|
||||
cls.base_url,
|
||||
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||
env={**os.environ, "SGLANG_ENABLE_JIT_DEEPGEMM": "False"},
|
||||
other_args=[
|
||||
"--attention-backend",
|
||||
"trtllm_mha",
|
||||
"--cuda-graph-max-bs",
|
||||
"512",
|
||||
"--tp-size",
|
||||
"4",
|
||||
"--ep-size",
|
||||
"4",
|
||||
"--mem-fraction-static",
|
||||
"0.7",
|
||||
"--mamba-ssm-dtype",
|
||||
"bfloat16",
|
||||
"--disable-radix-cache",
|
||||
],
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
kill_process_tree(cls.process.pid)
|
||||
|
||||
def test_gsm8k(self):
|
||||
args = SimpleNamespace(
|
||||
num_shots=5,
|
||||
data_path=None,
|
||||
num_questions=200,
|
||||
max_new_tokens=512,
|
||||
parallel=128,
|
||||
host="http://127.0.0.1",
|
||||
port=int(self.base_url.split(":")[-1]),
|
||||
)
|
||||
metrics = run_eval(args)
|
||||
print(f"{metrics=}")
|
||||
self.assertGreater(metrics["accuracy"], 0.93)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -1,65 +0,0 @@
|
||||
import os
|
||||
import unittest
|
||||
from types import SimpleNamespace
|
||||
|
||||
from sglang.srt.utils import kill_process_tree
|
||||
from sglang.test.few_shot_gsm8k import run_eval
|
||||
from sglang.test.test_utils import (
|
||||
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||
DEFAULT_URL_FOR_TEST,
|
||||
CustomTestCase,
|
||||
popen_launch_server,
|
||||
)
|
||||
|
||||
|
||||
class TestFlashinferTrtllmGenMoeBackend(CustomTestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.model = "Qwen/Qwen3-Next-80B-A3B-Instruct-FP8"
|
||||
cls.base_url = DEFAULT_URL_FOR_TEST
|
||||
cls.process = popen_launch_server(
|
||||
cls.model,
|
||||
cls.base_url,
|
||||
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||
env={**os.environ, "SGLANG_ENABLE_JIT_DEEPGEMM": "False"},
|
||||
other_args=[
|
||||
"--attention-backend",
|
||||
"triton",
|
||||
"--moe-runner-backend",
|
||||
"flashinfer_trtllm",
|
||||
"--cuda-graph-max-bs",
|
||||
"512",
|
||||
"--tp-size",
|
||||
"4",
|
||||
"--ep-size",
|
||||
"4",
|
||||
"--mem-fraction-static",
|
||||
"0.7",
|
||||
"--mamba-ssm-dtype",
|
||||
"bfloat16",
|
||||
"--quantization",
|
||||
"fp8",
|
||||
],
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
kill_process_tree(cls.process.pid)
|
||||
|
||||
def test_gsm8k(self):
|
||||
args = SimpleNamespace(
|
||||
num_shots=5,
|
||||
data_path=None,
|
||||
num_questions=200,
|
||||
max_new_tokens=512,
|
||||
parallel=128,
|
||||
host="http://127.0.0.1",
|
||||
port=int(self.base_url.split(":")[-1]),
|
||||
)
|
||||
metrics = run_eval(args)
|
||||
print(f"{metrics=}")
|
||||
self.assertGreater(metrics["accuracy"], 0.93)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -1,58 +0,0 @@
|
||||
import unittest
|
||||
|
||||
from nightly_utils import NightlyBenchmarkRunner
|
||||
|
||||
from sglang.test.test_utils import DEFAULT_URL_FOR_TEST
|
||||
|
||||
PROFILE_DIR = "performance_profiles_gpt_oss_4gpu"
|
||||
|
||||
|
||||
class TestNightlyGptOss4GpuPerformance(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.models = [
|
||||
(
|
||||
"openai/gpt-oss-120b",
|
||||
[
|
||||
"--tp",
|
||||
"4",
|
||||
"--cuda-graph-max-bs",
|
||||
"200",
|
||||
"--mem-fraction-static",
|
||||
"0.93",
|
||||
],
|
||||
),
|
||||
]
|
||||
cls.base_url = DEFAULT_URL_FOR_TEST
|
||||
cls.batch_sizes = [1, 1, 8, 16, 64]
|
||||
cls.input_lens = (4096,)
|
||||
cls.output_lens = (512,)
|
||||
cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url)
|
||||
cls.runner.setup_profile_directory()
|
||||
|
||||
def test_bench_one_batch(self):
|
||||
all_model_succeed = True
|
||||
|
||||
for model_path, other_args in self.models:
|
||||
with self.subTest(model=model_path):
|
||||
results, success = self.runner.run_benchmark_for_model(
|
||||
model_path=model_path,
|
||||
batch_sizes=self.batch_sizes,
|
||||
input_lens=self.input_lens,
|
||||
output_lens=self.output_lens,
|
||||
other_args=other_args,
|
||||
)
|
||||
|
||||
if not success:
|
||||
all_model_succeed = False
|
||||
|
||||
self.runner.add_report(results)
|
||||
|
||||
self.runner.write_final_report()
|
||||
|
||||
if not all_model_succeed:
|
||||
raise AssertionError("Some models failed the perf tests.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user