[CI] Migrate nightly tests to test/registered/ (#15582)
This commit is contained in:
Alison Shao
263
python/sglang/test/accuracy_test_runner.py
Normal file
263
python/sglang/test/accuracy_test_runner.py
Normal file
@@ -0,0 +1,263 @@
|
||||
from dataclasses import dataclass
|
||||
from types import SimpleNamespace
|
||||
from typing import List, Optional, Tuple
|
||||
|
||||
from sglang.srt.utils import kill_process_tree
|
||||
from sglang.test.run_eval import run_eval
|
||||
from sglang.test.test_utils import (
|
||||
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||
DEFAULT_URL_FOR_TEST,
|
||||
ModelLaunchSettings,
|
||||
popen_launch_server,
|
||||
write_github_step_summary,
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class AccuracyTestParams:
|
||||
"""Parameters for accuracy testing."""
|
||||
|
||||
dataset: str # e.g., "mgsm_en", "gsm8k", "mmmu", "gpqa"
|
||||
baseline_accuracy: float # Required: minimum accuracy threshold
|
||||
num_examples: Optional[int] = None
|
||||
num_threads: Optional[int] = None
|
||||
max_tokens: Optional[int] = None
|
||||
return_latency: bool = False
|
||||
# Extended parameters for special evaluations (e.g., GPQA with thinking mode)
|
||||
thinking_mode: Optional[str] = None # e.g., "deepseek-v3"
|
||||
temperature: Optional[float] = None
|
||||
repeat: Optional[int] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class AccuracyTestResult:
|
||||
"""Result of an accuracy test."""
|
||||
|
||||
model: str
|
||||
dataset: str
|
||||
passed: bool
|
||||
score: Optional[float]
|
||||
baseline_accuracy: float
|
||||
error: Optional[str]
|
||||
latency: Optional[float] = None
|
||||
|
||||
|
||||
def write_accuracy_github_summary(
|
||||
test_name: str,
|
||||
dataset: str,
|
||||
results: List[AccuracyTestResult],
|
||||
) -> None:
|
||||
"""Write accuracy test results to GitHub step summary.
|
||||
|
||||
Args:
|
||||
test_name: Name of the test
|
||||
dataset: Dataset name used for evaluation
|
||||
results: List of AccuracyTestResult objects
|
||||
"""
|
||||
summary = f"## {test_name} - Accuracy ({dataset})\n"
|
||||
summary += "| model | status | score | baseline | error |\n"
|
||||
summary += "| ----- | ------ | ----- | -------- | ----- |\n"
|
||||
|
||||
for result in results:
|
||||
status_emoji = "✅" if result.passed else "❌"
|
||||
score_str = f"{result.score:.4f}" if result.score is not None else "N/A"
|
||||
baseline_str = f"{result.baseline_accuracy:.4f}"
|
||||
error_str = result.error if result.error else "-"
|
||||
summary += f"| {result.model} | {status_emoji} | {score_str} | {baseline_str} | {error_str} |\n"
|
||||
|
||||
write_github_step_summary(summary)
|
||||
|
||||
|
||||
def _run_simple_eval(
|
||||
model: ModelLaunchSettings,
|
||||
base_url: str,
|
||||
dataset: str,
|
||||
num_examples: Optional[int] = None,
|
||||
num_threads: Optional[int] = None,
|
||||
max_tokens: Optional[int] = None,
|
||||
return_latency: bool = False,
|
||||
thinking_mode: Optional[str] = None,
|
||||
temperature: Optional[float] = None,
|
||||
repeat: Optional[int] = None,
|
||||
) -> Tuple[bool, Optional[str], Optional[dict]]:
|
||||
"""Run evaluation using simple_eval backend (run_eval.py).
|
||||
|
||||
Returns:
|
||||
Tuple of (success, error_message, metrics_dict)
|
||||
"""
|
||||
process = None
|
||||
try:
|
||||
process = popen_launch_server(
|
||||
model.model_path,
|
||||
base_url,
|
||||
other_args=model.extra_args,
|
||||
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||
)
|
||||
|
||||
args = SimpleNamespace(
|
||||
base_url=base_url,
|
||||
model=model.model_path,
|
||||
eval_name=dataset,
|
||||
num_examples=num_examples,
|
||||
num_threads=num_threads or 1024,
|
||||
)
|
||||
|
||||
if max_tokens is not None:
|
||||
args.max_tokens = max_tokens
|
||||
|
||||
if return_latency:
|
||||
args.return_latency = True
|
||||
|
||||
if thinking_mode is not None:
|
||||
args.thinking_mode = thinking_mode
|
||||
|
||||
if temperature is not None:
|
||||
args.temperature = temperature
|
||||
|
||||
if repeat is not None:
|
||||
args.repeat = repeat
|
||||
|
||||
result = run_eval(args)
|
||||
|
||||
# Handle result format (run_eval can return metrics or (metrics, latency))
|
||||
if return_latency and isinstance(result, tuple):
|
||||
metrics, latency = result
|
||||
metrics["latency"] = round(latency, 4)
|
||||
else:
|
||||
metrics = result
|
||||
|
||||
return True, None, metrics
|
||||
|
||||
except Exception as e:
|
||||
return False, f"Accuracy test exception: {str(e)}", None
|
||||
|
||||
finally:
|
||||
if process:
|
||||
kill_process_tree(process.pid)
|
||||
|
||||
|
||||
def _run_few_shot_eval(
|
||||
model: ModelLaunchSettings,
|
||||
base_url: str,
|
||||
num_questions: Optional[int] = None,
|
||||
num_shots: int = 8,
|
||||
max_tokens: int = 512,
|
||||
) -> Tuple[bool, Optional[str], Optional[dict]]:
|
||||
"""Run evaluation using few_shot backend (few_shot_gsm8k.py).
|
||||
|
||||
Returns:
|
||||
Tuple of (success, error_message, metrics_dict)
|
||||
"""
|
||||
from sglang.test.few_shot_gsm8k import run_eval as run_few_shot_eval
|
||||
|
||||
process = None
|
||||
try:
|
||||
process = popen_launch_server(
|
||||
model.model_path,
|
||||
base_url,
|
||||
other_args=model.extra_args,
|
||||
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||
)
|
||||
|
||||
args = SimpleNamespace(
|
||||
num_shots=num_shots,
|
||||
data_path=None,
|
||||
num_questions=num_questions or 200,
|
||||
max_new_tokens=max_tokens,
|
||||
parallel=128,
|
||||
host="http://127.0.0.1",
|
||||
port=int(base_url.split(":")[-1]),
|
||||
)
|
||||
|
||||
metrics = run_few_shot_eval(args)
|
||||
|
||||
# Normalize metrics format (few_shot returns "accuracy", simple_eval returns "score")
|
||||
if "accuracy" in metrics and "score" not in metrics:
|
||||
metrics["score"] = metrics["accuracy"]
|
||||
|
||||
return True, None, metrics
|
||||
|
||||
except Exception as e:
|
||||
return False, f"Few-shot evaluation exception: {str(e)}", None
|
||||
|
||||
finally:
|
||||
if process:
|
||||
kill_process_tree(process.pid)
|
||||
|
||||
|
||||
def run_accuracy_test(
|
||||
model: ModelLaunchSettings,
|
||||
params: AccuracyTestParams,
|
||||
base_url: Optional[str] = None,
|
||||
) -> AccuracyTestResult:
|
||||
"""Run accuracy test for a single model.
|
||||
|
||||
Args:
|
||||
model: ModelLaunchSettings with model config
|
||||
params: AccuracyTestParams with dataset, baseline, and optional settings
|
||||
base_url: Server base URL (default: DEFAULT_URL_FOR_TEST)
|
||||
|
||||
Returns:
|
||||
AccuracyTestResult with test outcome
|
||||
"""
|
||||
base_url = base_url or DEFAULT_URL_FOR_TEST
|
||||
|
||||
print(f"\n{'='*60}")
|
||||
print(f"Running ACCURACY test for {model.model_path}")
|
||||
print(f" Dataset: {params.dataset}")
|
||||
print(f" Baseline: {params.baseline_accuracy}")
|
||||
print(f"{'='*60}\n")
|
||||
|
||||
# Run evaluation based on dataset type
|
||||
if params.dataset == "gsm8k":
|
||||
success, error, metrics = _run_few_shot_eval(
|
||||
model=model,
|
||||
base_url=base_url,
|
||||
num_questions=params.num_examples,
|
||||
max_tokens=params.max_tokens or 512,
|
||||
)
|
||||
else:
|
||||
success, error, metrics = _run_simple_eval(
|
||||
model=model,
|
||||
base_url=base_url,
|
||||
dataset=params.dataset,
|
||||
num_examples=params.num_examples,
|
||||
num_threads=params.num_threads,
|
||||
max_tokens=params.max_tokens,
|
||||
return_latency=params.return_latency,
|
||||
thinking_mode=params.thinking_mode,
|
||||
temperature=params.temperature,
|
||||
repeat=params.repeat,
|
||||
)
|
||||
|
||||
if not success:
|
||||
print(f"✗ Accuracy test failed for {model.model_path}: {error}")
|
||||
return AccuracyTestResult(
|
||||
model=model.model_path,
|
||||
dataset=params.dataset,
|
||||
passed=False,
|
||||
score=None,
|
||||
baseline_accuracy=params.baseline_accuracy,
|
||||
error=error,
|
||||
)
|
||||
|
||||
# Validate against baseline
|
||||
score = metrics.get("score", 0.0)
|
||||
passed = score >= params.baseline_accuracy
|
||||
latency = metrics.get("latency")
|
||||
|
||||
if passed:
|
||||
print(f"✓ Accuracy {score:.3f} >= baseline {params.baseline_accuracy:.3f}")
|
||||
else:
|
||||
error = f"Accuracy {score:.3f} below baseline {params.baseline_accuracy:.3f}"
|
||||
print(f"✗ {error}")
|
||||
|
||||
return AccuracyTestResult(
|
||||
model=model.model_path,
|
||||
dataset=params.dataset,
|
||||
passed=passed,
|
||||
score=score,
|
||||
baseline_accuracy=params.baseline_accuracy,
|
||||
error=error if not passed else None,
|
||||
latency=latency,
|
||||
)
|
||||
0
python/sglang/test/ascend/__init__.py
Normal file
0
python/sglang/test/ascend/__init__.py
Normal file
68
python/sglang/test/ascend/gsm8k_ascend_mixin.py
Normal file
68
python/sglang/test/ascend/gsm8k_ascend_mixin.py
Normal file
@@ -0,0 +1,68 @@
|
||||
import os
|
||||
from abc import ABC
|
||||
from types import SimpleNamespace
|
||||
|
||||
from sglang.srt.utils import kill_process_tree
|
||||
from sglang.test.few_shot_gsm8k import run_eval
|
||||
from sglang.test.test_utils import (
|
||||
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||
DEFAULT_URL_FOR_TEST,
|
||||
popen_launch_server,
|
||||
)
|
||||
|
||||
|
||||
class GSM8KAscendMixin(ABC):
|
||||
model = ""
|
||||
accuracy = 0.00
|
||||
other_args = [
|
||||
"--trust-remote-code",
|
||||
"--mem-fraction-static",
|
||||
"0.8",
|
||||
"--attention-backend",
|
||||
"ascend",
|
||||
"--disable-cuda-graph",
|
||||
]
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.base_url = DEFAULT_URL_FOR_TEST
|
||||
os.environ["PYTORCH_NPU_ALLOC_CONF"] = "expandable_segments:True"
|
||||
os.environ["ASCEND_MF_STORE_URL"] = "tcp://127.0.0.1:24666"
|
||||
os.environ["HCCL_BUFFSIZE"] = "200"
|
||||
os.environ["SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK"] = "24"
|
||||
os.environ["USE_VLLM_CUSTOM_ALLREDUCE"] = "1"
|
||||
os.environ["HCCL_EXEC_TIMEOUT"] = "200"
|
||||
os.environ["STREAMS_PER_DEVICE"] = "32"
|
||||
os.environ["SGLANG_ENBLE_TORCH_COMILE"] = "1"
|
||||
os.environ["AUTO_USE_UC_MEMORY"] = "0"
|
||||
os.environ["P2P_HCCL_BUFFSIZE"] = "20"
|
||||
env = os.environ.copy()
|
||||
|
||||
cls.process = popen_launch_server(
|
||||
cls.model,
|
||||
cls.base_url,
|
||||
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||
other_args=cls.other_args,
|
||||
env=env,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
kill_process_tree(cls.process.pid)
|
||||
|
||||
def test_gsm8k(self):
|
||||
args = SimpleNamespace(
|
||||
num_shots=5,
|
||||
data_path=None,
|
||||
num_questions=200,
|
||||
max_new_tokens=512,
|
||||
parallel=128,
|
||||
host="http://127.0.0.1",
|
||||
port=int(self.base_url.split(":")[-1]),
|
||||
)
|
||||
metrics = run_eval(args)
|
||||
self.assertGreater(
|
||||
metrics["accuracy"],
|
||||
self.accuracy,
|
||||
f'Accuracy of {self.model} is {str(metrics["accuracy"])}, is lower than {self.accuracy}',
|
||||
)
|
||||
217
python/sglang/test/ascend/vlm_utils.py
Normal file
217
python/sglang/test/ascend/vlm_utils.py
Normal file
@@ -0,0 +1,217 @@
|
||||
import glob
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
|
||||
from sglang.srt.utils import kill_process_tree
|
||||
from sglang.test.test_utils import (
|
||||
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||
DEFAULT_URL_FOR_TEST,
|
||||
CustomTestCase,
|
||||
popen_launch_server,
|
||||
)
|
||||
|
||||
|
||||
class TestVLMModels(CustomTestCase):
|
||||
model = ""
|
||||
mmmu_accuracy = 0.00
|
||||
other_args = [
|
||||
"--trust-remote-code",
|
||||
"--cuda-graph-max-bs",
|
||||
"32",
|
||||
"--enable-multimodal",
|
||||
"--mem-fraction-static",
|
||||
0.35,
|
||||
"--log-level",
|
||||
"info",
|
||||
"--attention-backend",
|
||||
"ascend",
|
||||
"--disable-cuda-graph",
|
||||
"--tp-size",
|
||||
4,
|
||||
]
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
# Removed argument parsing from here
|
||||
cls.base_url = DEFAULT_URL_FOR_TEST
|
||||
cls.api_key = "sk-123456"
|
||||
cls.time_out = DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH
|
||||
|
||||
# Set OpenAI API key and base URL environment variables. Needed for lmm-evals to work.
|
||||
os.environ["OPENAI_API_KEY"] = cls.api_key
|
||||
os.environ["OPENAI_API_BASE"] = f"{cls.base_url}/v1"
|
||||
|
||||
def run_mmmu_eval(
|
||||
self,
|
||||
model_version: str,
|
||||
output_path: str,
|
||||
limit: str,
|
||||
*,
|
||||
env: dict | None = None,
|
||||
):
|
||||
"""
|
||||
Evaluate a VLM on the MMMU validation set with lmms‑eval.
|
||||
Only `model_version` (checkpoint) and `chat_template` vary;
|
||||
We are focusing only on the validation set due to resource constraints.
|
||||
"""
|
||||
# -------- fixed settings --------
|
||||
model = "openai_compatible"
|
||||
tp = 1
|
||||
tasks = "mmmu_val"
|
||||
batch_size = 2
|
||||
log_suffix = "openai_compatible"
|
||||
os.makedirs(output_path, exist_ok=True)
|
||||
|
||||
# -------- compose --model_args --------
|
||||
model_args = f'model_version="{model_version}",' f"tp={tp}"
|
||||
|
||||
# -------- build command list --------
|
||||
cmd = [
|
||||
"python3",
|
||||
"-m",
|
||||
"lmms_eval",
|
||||
"--model",
|
||||
model,
|
||||
"--model_args",
|
||||
model_args,
|
||||
"--tasks",
|
||||
tasks,
|
||||
"--batch_size",
|
||||
str(batch_size),
|
||||
"--log_samples",
|
||||
"--log_samples_suffix",
|
||||
log_suffix,
|
||||
"--output_path",
|
||||
str(output_path),
|
||||
"--limit",
|
||||
limit,
|
||||
"--config",
|
||||
"/__w/sglang/sglang/test/registered/ascend/vlm_models/mmmu-val.yaml",
|
||||
]
|
||||
|
||||
subprocess.run(
|
||||
cmd,
|
||||
check=True,
|
||||
timeout=3600,
|
||||
)
|
||||
|
||||
def _run_vlm_mmmu_test(
|
||||
self,
|
||||
output_path="./logs",
|
||||
test_name="",
|
||||
custom_env=None,
|
||||
capture_output=False,
|
||||
limit="50",
|
||||
):
|
||||
"""
|
||||
Common method to run VLM MMMU benchmark test.
|
||||
Args:
|
||||
model: Model to test
|
||||
output_path: Path for output logs
|
||||
test_name: Optional test name for logging
|
||||
custom_env: Optional custom environment variables
|
||||
capture_output: Whether to capture server stdout/stderr
|
||||
"""
|
||||
print(f"\nTesting model: {self.model}{test_name}")
|
||||
|
||||
process = None
|
||||
server_output = ""
|
||||
|
||||
try:
|
||||
# Prepare environment variables
|
||||
process_env = os.environ.copy()
|
||||
if custom_env:
|
||||
process_env.update(custom_env)
|
||||
|
||||
# Prepare stdout/stderr redirection if needed
|
||||
stdout_file = None
|
||||
stderr_file = None
|
||||
if capture_output:
|
||||
stdout_file = open("/tmp/server_stdout.log", "w")
|
||||
stderr_file = open("/tmp/server_stderr.log", "w")
|
||||
|
||||
process = popen_launch_server(
|
||||
self.model,
|
||||
base_url=self.base_url,
|
||||
timeout=self.time_out,
|
||||
api_key=self.api_key,
|
||||
other_args=self.other_args,
|
||||
env=process_env,
|
||||
return_stdout_stderr=(
|
||||
(stdout_file, stderr_file) if capture_output else None
|
||||
),
|
||||
)
|
||||
|
||||
# Run evaluation
|
||||
self.run_mmmu_eval(self.model, output_path, limit)
|
||||
|
||||
# Get the result file
|
||||
result_file_path = glob.glob(f"{output_path}/*.json")[0]
|
||||
|
||||
with open(result_file_path, "r") as f:
|
||||
result = json.load(f)
|
||||
print(f"Result{test_name}\n: {result}")
|
||||
|
||||
# Process the result
|
||||
mmmu_accuracy = result["results"]["mmmu_val"]["mmmu_acc,none"]
|
||||
print(
|
||||
f"Model {self.model} achieved accuracy{test_name}: {mmmu_accuracy:.4f}"
|
||||
)
|
||||
|
||||
# Capture server output if requested
|
||||
if capture_output and process:
|
||||
server_output = self._read_output_from_files()
|
||||
|
||||
# Assert performance meets expected threshold
|
||||
self.assertGreaterEqual(
|
||||
mmmu_accuracy,
|
||||
self.mmmu_accuracy,
|
||||
f"Model {self.model} accuracy ({mmmu_accuracy:.4f}) below expected threshold ({self.mmmu_accuracy:.4f}){test_name}",
|
||||
)
|
||||
|
||||
return server_output
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error testing {self.model}{test_name}: {e}")
|
||||
self.fail(f"Test failed for {self.model}{test_name}: {e}")
|
||||
|
||||
finally:
|
||||
# Ensure process cleanup happens regardless of success/failure
|
||||
if process is not None and process.poll() is None:
|
||||
print(f"Cleaning up process {process.pid}")
|
||||
try:
|
||||
kill_process_tree(process.pid)
|
||||
except Exception as e:
|
||||
print(f"Error killing process: {e}")
|
||||
|
||||
# clean up temporary files
|
||||
if capture_output:
|
||||
if stdout_file:
|
||||
stdout_file.close()
|
||||
if stderr_file:
|
||||
stderr_file.close()
|
||||
for filename in ["/tmp/server_stdout.log", "/tmp/server_stderr.log"]:
|
||||
try:
|
||||
if os.path.exists(filename):
|
||||
os.remove(filename)
|
||||
except Exception as e:
|
||||
print(f"Error removing {filename}: {e}")
|
||||
|
||||
def _read_output_from_files(self):
|
||||
output_lines = []
|
||||
|
||||
log_files = [
|
||||
("/tmp/server_stdout.log", "[STDOUT]"),
|
||||
("/tmp/server_stderr.log", "[STDERR]"),
|
||||
]
|
||||
for filename, tag in log_files:
|
||||
try:
|
||||
if os.path.exists(filename):
|
||||
with open(filename, "r") as f:
|
||||
for line in f:
|
||||
output_lines.append(f"{tag} {line.rstrip()}")
|
||||
except Exception as e:
|
||||
print(f"Error reading {tag.lower()} file: {e}")
|
||||
|
||||
return "\n".join(output_lines)
|
||||
333
python/sglang/test/nightly_utils.py
Normal file
333
python/sglang/test/nightly_utils.py
Normal file
@@ -0,0 +1,333 @@
|
||||
"""Utilities for running nightly performance benchmarks with profiling."""
|
||||
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import time
|
||||
from typing import List, Optional, Tuple
|
||||
|
||||
import requests
|
||||
|
||||
from sglang.srt.utils import kill_process_tree
|
||||
from sglang.test.nightly_bench_utils import BenchmarkResult, generate_markdown_report
|
||||
from sglang.test.test_utils import (
|
||||
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||
is_in_ci,
|
||||
popen_launch_server,
|
||||
write_github_step_summary,
|
||||
)
|
||||
|
||||
|
||||
class NightlyBenchmarkRunner:
|
||||
"""Helper class for running nightly performance benchmarks with profiling.
|
||||
|
||||
This class encapsulates common patterns used across nightly performance tests,
|
||||
including profile directory management, benchmark command construction,
|
||||
result parsing, and report generation.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
profile_dir: str,
|
||||
test_name: str,
|
||||
base_url: str,
|
||||
gpu_config: str = None,
|
||||
):
|
||||
"""Initialize the benchmark runner.
|
||||
|
||||
Args:
|
||||
profile_dir: Directory to store performance profiles
|
||||
test_name: Name of the test (used for reporting)
|
||||
base_url: Base URL for the server
|
||||
gpu_config: Optional GPU configuration string (e.g., "2-gpu-h100", "8-gpu-b200")
|
||||
"""
|
||||
self.profile_dir = profile_dir
|
||||
self.test_name = test_name
|
||||
self.base_url = base_url
|
||||
self.gpu_config = gpu_config or os.environ.get("GPU_CONFIG", "")
|
||||
|
||||
# Include GPU config in report header if available
|
||||
header = f"## {test_name}"
|
||||
if self.gpu_config:
|
||||
header += f" ({self.gpu_config})"
|
||||
header += "\n"
|
||||
self.full_report = header + BenchmarkResult.help_str()
|
||||
|
||||
def setup_profile_directory(self) -> None:
|
||||
"""Create the profile directory if it doesn't exist."""
|
||||
os.makedirs(self.profile_dir, exist_ok=True)
|
||||
|
||||
def generate_profile_filename(
|
||||
self, model_path: str, variant: str = ""
|
||||
) -> Tuple[str, str]:
|
||||
"""Generate unique profile filename and path for the model.
|
||||
|
||||
Args:
|
||||
model_path: Path to the model (e.g., "deepseek-ai/DeepSeek-V3.1")
|
||||
variant: Optional variant suffix (e.g., "basic", "mtp", "nsa")
|
||||
|
||||
Returns:
|
||||
Tuple of (profile_path_prefix, json_output_file)
|
||||
"""
|
||||
timestamp = int(time.time())
|
||||
model_safe_name = model_path.replace("/", "_")
|
||||
|
||||
# Build filename with optional variant
|
||||
if variant:
|
||||
profile_filename = f"{model_safe_name}_{variant}_{timestamp}"
|
||||
json_filename = f"results_{model_safe_name}_{variant}_{timestamp}.json"
|
||||
else:
|
||||
profile_filename = f"{model_safe_name}_{timestamp}"
|
||||
json_filename = f"results_{model_safe_name}_{timestamp}.json"
|
||||
|
||||
profile_path_prefix = os.path.join(self.profile_dir, profile_filename)
|
||||
|
||||
return profile_path_prefix, json_filename
|
||||
|
||||
def build_benchmark_command(
|
||||
self,
|
||||
model_path: str,
|
||||
batch_sizes: List[int],
|
||||
input_lens: Tuple[int, ...],
|
||||
output_lens: Tuple[int, ...],
|
||||
profile_path_prefix: str,
|
||||
json_output_file: str,
|
||||
extra_args: Optional[List[str]] = None,
|
||||
) -> List[str]:
|
||||
"""Build the benchmark command with all required arguments.
|
||||
|
||||
Args:
|
||||
model_path: Path to the model
|
||||
batch_sizes: List of batch sizes to test
|
||||
input_lens: Tuple of input lengths to test
|
||||
output_lens: Tuple of output lengths to test
|
||||
profile_path_prefix: Prefix for profile output files
|
||||
json_output_file: Path to JSON output file
|
||||
extra_args: Optional extra arguments to append to command
|
||||
|
||||
Returns:
|
||||
List of command arguments ready for subprocess.run()
|
||||
"""
|
||||
command = [
|
||||
"python3",
|
||||
"-m",
|
||||
"sglang.bench_one_batch_server",
|
||||
"--model",
|
||||
model_path,
|
||||
"--base-url",
|
||||
self.base_url,
|
||||
"--batch-size",
|
||||
*[str(x) for x in batch_sizes],
|
||||
"--input-len",
|
||||
*[str(x) for x in input_lens],
|
||||
"--output-len",
|
||||
*[str(x) for x in output_lens],
|
||||
"--show-report",
|
||||
"--profile",
|
||||
"--profile-by-stage",
|
||||
"--profile-output-dir",
|
||||
profile_path_prefix,
|
||||
f"--pydantic-result-filename={json_output_file}",
|
||||
"--no-append-to-github-summary",
|
||||
]
|
||||
|
||||
if extra_args:
|
||||
command.extend(extra_args)
|
||||
|
||||
return command
|
||||
|
||||
def run_benchmark_command(
|
||||
self, command: List[str], model_description: str = ""
|
||||
) -> Tuple[subprocess.CompletedProcess, bool]:
|
||||
"""Execute the benchmark command and return the result.
|
||||
|
||||
Args:
|
||||
command: Command to execute
|
||||
model_description: Description for logging (e.g., "model_name (variant)")
|
||||
|
||||
Returns:
|
||||
Tuple of (CompletedProcess, success_bool)
|
||||
"""
|
||||
print(f"Running command: {' '.join(command)}")
|
||||
result = subprocess.run(command, capture_output=True, text=True)
|
||||
|
||||
if result.returncode != 0:
|
||||
desc = model_description or "benchmark"
|
||||
print(f"Error running benchmark for {desc}:")
|
||||
print(result.stderr)
|
||||
return result, False
|
||||
|
||||
return result, True
|
||||
|
||||
def load_benchmark_results(
|
||||
self, json_output_file: str, model_description: str = ""
|
||||
) -> Tuple[List[BenchmarkResult], bool]:
|
||||
"""Load and parse benchmark results from JSON file.
|
||||
|
||||
Args:
|
||||
json_output_file: Path to JSON output file
|
||||
model_description: Description for logging
|
||||
|
||||
Returns:
|
||||
Tuple of (list of BenchmarkResult objects, success_bool)
|
||||
"""
|
||||
benchmark_results = []
|
||||
|
||||
if not os.path.exists(json_output_file):
|
||||
desc = model_description or "model"
|
||||
print(f"Warning: JSON output file {json_output_file} not found for {desc}")
|
||||
return benchmark_results, False
|
||||
|
||||
try:
|
||||
with open(json_output_file, "r") as f:
|
||||
json_data = json.load(f)
|
||||
|
||||
# Convert JSON data to BenchmarkResult objects
|
||||
for data in json_data:
|
||||
benchmark_result = BenchmarkResult(**data)
|
||||
benchmark_results.append(benchmark_result)
|
||||
|
||||
print(
|
||||
f"Loaded {len(benchmark_results)} benchmark results from {json_output_file}"
|
||||
)
|
||||
|
||||
# Clean up JSON file
|
||||
os.remove(json_output_file)
|
||||
|
||||
return benchmark_results, True
|
||||
|
||||
except Exception as e:
|
||||
desc = model_description or "model"
|
||||
print(f"Error loading benchmark results for {desc}: {e}")
|
||||
# Try to clean up the file anyway
|
||||
if os.path.exists(json_output_file):
|
||||
os.remove(json_output_file)
|
||||
return benchmark_results, False
|
||||
|
||||
def run_benchmark_for_model(
|
||||
self,
|
||||
model_path: str,
|
||||
batch_sizes: List[int],
|
||||
input_lens: Tuple[int, ...],
|
||||
output_lens: Tuple[int, ...],
|
||||
other_args: Optional[List[str]] = None,
|
||||
variant: str = "",
|
||||
extra_bench_args: Optional[List[str]] = None,
|
||||
) -> Tuple[List[BenchmarkResult], bool, Optional[float]]:
|
||||
"""Run a complete benchmark for a single model with server management.
|
||||
|
||||
This method handles:
|
||||
- Server launch and cleanup
|
||||
- Profile filename generation
|
||||
- Benchmark command construction and execution
|
||||
- Result loading and parsing
|
||||
- Fetching speculative decoding accept length (for MTP/EAGLE)
|
||||
|
||||
Args:
|
||||
model_path: Path to the model
|
||||
batch_sizes: List of batch sizes to test
|
||||
input_lens: Tuple of input lengths
|
||||
output_lens: Tuple of output lengths
|
||||
other_args: Arguments to pass to server launch
|
||||
variant: Optional variant suffix (e.g., "basic", "mtp")
|
||||
extra_bench_args: Extra arguments for the benchmark command
|
||||
|
||||
Returns:
|
||||
Tuple of (list of BenchmarkResult objects, success_bool, avg_spec_accept_length or None)
|
||||
"""
|
||||
benchmark_results = []
|
||||
avg_spec_accept_length = None
|
||||
model_description = f"{model_path}" + (f" ({variant})" if variant else "")
|
||||
|
||||
# Launch server
|
||||
process = popen_launch_server(
|
||||
model=model_path,
|
||||
base_url=self.base_url,
|
||||
other_args=other_args or [],
|
||||
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||
)
|
||||
|
||||
try:
|
||||
# Generate filenames
|
||||
profile_path_prefix, json_output_file = self.generate_profile_filename(
|
||||
model_path, variant
|
||||
)
|
||||
|
||||
# Build and run benchmark command
|
||||
# Prepare extra args with run_name if variant is specified
|
||||
bench_args = list(extra_bench_args) if extra_bench_args else []
|
||||
if variant:
|
||||
bench_args.extend(["--run-name", variant])
|
||||
|
||||
command = self.build_benchmark_command(
|
||||
model_path,
|
||||
batch_sizes,
|
||||
input_lens,
|
||||
output_lens,
|
||||
profile_path_prefix,
|
||||
json_output_file,
|
||||
extra_args=bench_args,
|
||||
)
|
||||
|
||||
result, cmd_success = self.run_benchmark_command(command, model_description)
|
||||
|
||||
if not cmd_success:
|
||||
return benchmark_results, False, None
|
||||
|
||||
# Load results
|
||||
benchmark_results, load_success = self.load_benchmark_results(
|
||||
json_output_file, model_description
|
||||
)
|
||||
|
||||
# Fetch speculative decoding accept length before killing server
|
||||
avg_spec_accept_length = self._get_spec_accept_length()
|
||||
|
||||
return benchmark_results, load_success, avg_spec_accept_length
|
||||
|
||||
finally:
|
||||
# Always clean up server process
|
||||
kill_process_tree(process.pid)
|
||||
|
||||
def _get_spec_accept_length(self) -> Optional[float]:
|
||||
"""Query the server for avg_spec_accept_length metric.
|
||||
|
||||
Returns:
|
||||
The average speculative decoding accept length, or None if not available.
|
||||
"""
|
||||
try:
|
||||
response = requests.get(f"{self.base_url}/get_server_info", timeout=10)
|
||||
if response.status_code == 200:
|
||||
server_info = response.json()
|
||||
internal_states = server_info.get("internal_states", [])
|
||||
if internal_states and len(internal_states) > 0:
|
||||
accept_length = internal_states[0].get("avg_spec_accept_length")
|
||||
if accept_length is not None:
|
||||
print(f" avg_spec_accept_length={accept_length:.2f}")
|
||||
return accept_length
|
||||
except Exception as e:
|
||||
print(f" Warning: Could not fetch spec accept length: {e}")
|
||||
return None
|
||||
|
||||
def add_report(self, results: List[BenchmarkResult]) -> None:
|
||||
"""Add benchmark results to the full report.
|
||||
|
||||
Args:
|
||||
results: List of BenchmarkResult objects to add to report
|
||||
"""
|
||||
if results:
|
||||
report_part = generate_markdown_report(self.profile_dir, results)
|
||||
self.full_report += report_part + "\n"
|
||||
|
||||
def write_final_report(self) -> None:
|
||||
"""Write the final report to GitHub summary if in CI."""
|
||||
if is_in_ci():
|
||||
write_github_step_summary(self.full_report)
|
||||
print(self.full_report)
|
||||
|
||||
def get_full_report(self) -> str:
|
||||
"""Get the accumulated full report.
|
||||
|
||||
Returns:
|
||||
The full markdown report as a string
|
||||
"""
|
||||
return self.full_report
|
||||
230
python/sglang/test/performance_test_runner.py
Normal file
230
python/sglang/test/performance_test_runner.py
Normal file
@@ -0,0 +1,230 @@
|
||||
from dataclasses import dataclass, field
|
||||
from typing import List, Optional, Tuple
|
||||
|
||||
from sglang.test.nightly_bench_utils import BenchmarkResult
|
||||
from sglang.test.nightly_utils import NightlyBenchmarkRunner
|
||||
from sglang.test.test_utils import DEFAULT_URL_FOR_TEST, ModelLaunchSettings
|
||||
|
||||
|
||||
@dataclass
|
||||
class PerformanceTestParams:
|
||||
"""Parameters for performance testing."""
|
||||
|
||||
batch_sizes: List[int] = field(default_factory=lambda: [1, 8, 16, 64])
|
||||
input_lens: Tuple[int, ...] = (4096,)
|
||||
output_lens: Tuple[int, ...] = (512,)
|
||||
profile_dir: Optional[str] = None # None = auto-generate based on is_vlm
|
||||
dataset_name: str = "mmmu" # For VLM perf test
|
||||
# MTP/EAGLE speculative decoding: minimum accept length threshold (None = no validation)
|
||||
spec_accept_length_threshold: Optional[float] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class PerformanceTestResult:
|
||||
"""Result of a performance test.
|
||||
|
||||
Aggregates metrics across all batch sizes tested for a single model.
|
||||
"""
|
||||
|
||||
model: str
|
||||
passed: bool
|
||||
error: Optional[str]
|
||||
# Aggregate metrics (from the largest batch size result, or None if failed)
|
||||
latency: Optional[float] = None
|
||||
input_throughput: Optional[float] = None
|
||||
output_throughput: Optional[float] = None
|
||||
overall_throughput: Optional[float] = None
|
||||
# All individual benchmark results
|
||||
benchmark_results: Optional[List[BenchmarkResult]] = None
|
||||
# MTP/EAGLE speculative decoding metric
|
||||
avg_spec_accept_length: Optional[float] = None
|
||||
|
||||
|
||||
def run_performance_test(
|
||||
model: ModelLaunchSettings,
|
||||
perf_runner: NightlyBenchmarkRunner,
|
||||
batch_sizes: List[int] = None,
|
||||
input_lens: Tuple[int, ...] = (4096,),
|
||||
output_lens: Tuple[int, ...] = (512,),
|
||||
is_vlm: bool = False,
|
||||
dataset_name: str = "mmmu",
|
||||
spec_accept_length_threshold: Optional[float] = None,
|
||||
) -> PerformanceTestResult:
|
||||
|
||||
# Set default for mutable argument
|
||||
if batch_sizes is None:
|
||||
batch_sizes = [1, 8, 16, 64]
|
||||
|
||||
print(f"\n{'='*60}")
|
||||
print(f"Running PERFORMANCE test for {model.model_path}")
|
||||
print(f" Batch sizes: {batch_sizes}")
|
||||
print(f" Input lens: {input_lens}")
|
||||
print(f" Output lens: {output_lens}")
|
||||
if spec_accept_length_threshold is not None:
|
||||
print(f" Spec accept length threshold: {spec_accept_length_threshold}")
|
||||
print(f"{'='*60}\n")
|
||||
|
||||
# Build extra args for benchmarks
|
||||
extra_bench_args = ["--trust-remote-code"]
|
||||
if is_vlm:
|
||||
extra_bench_args.append(f"--dataset-name={dataset_name}")
|
||||
|
||||
try:
|
||||
results, success, avg_spec_accept_length = perf_runner.run_benchmark_for_model(
|
||||
model_path=model.model_path,
|
||||
batch_sizes=batch_sizes,
|
||||
input_lens=input_lens,
|
||||
output_lens=output_lens,
|
||||
other_args=model.extra_args,
|
||||
extra_bench_args=extra_bench_args,
|
||||
)
|
||||
|
||||
if success and results:
|
||||
perf_runner.add_report(results)
|
||||
print(f"✓ Performance test succeeded for {model.model_path}")
|
||||
|
||||
# Validate speculative decoding accept length if threshold is set
|
||||
error_msg = None
|
||||
passed = True
|
||||
if spec_accept_length_threshold is not None:
|
||||
if avg_spec_accept_length is None:
|
||||
error_msg = f"Spec accept length threshold set but no accept length reported"
|
||||
passed = False
|
||||
print(f"✗ {error_msg}")
|
||||
elif avg_spec_accept_length < spec_accept_length_threshold:
|
||||
error_msg = (
|
||||
f"Spec accept length {avg_spec_accept_length:.2f} < "
|
||||
f"threshold {spec_accept_length_threshold}"
|
||||
)
|
||||
passed = False
|
||||
print(f"✗ {error_msg}")
|
||||
else:
|
||||
print(
|
||||
f"✓ Spec accept length {avg_spec_accept_length:.2f} >= "
|
||||
f"threshold {spec_accept_length_threshold}"
|
||||
)
|
||||
|
||||
# Extract aggregate metrics from the largest batch size result
|
||||
largest_batch_result = max(results, key=lambda r: r.batch_size)
|
||||
return PerformanceTestResult(
|
||||
model=model.model_path,
|
||||
passed=passed,
|
||||
error=error_msg,
|
||||
latency=largest_batch_result.latency,
|
||||
input_throughput=largest_batch_result.input_throughput,
|
||||
output_throughput=largest_batch_result.output_throughput,
|
||||
overall_throughput=largest_batch_result.overall_throughput,
|
||||
benchmark_results=results,
|
||||
avg_spec_accept_length=avg_spec_accept_length,
|
||||
)
|
||||
else:
|
||||
error_msg = f"Performance test failed for {model.model_path}"
|
||||
print(f"✗ {error_msg}")
|
||||
return PerformanceTestResult(
|
||||
model=model.model_path,
|
||||
passed=False,
|
||||
error=error_msg,
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
error_msg = f"Performance test exception for {model.model_path}: {str(e)}"
|
||||
print(f"✗ {error_msg}")
|
||||
return PerformanceTestResult(
|
||||
model=model.model_path,
|
||||
passed=False,
|
||||
error=error_msg,
|
||||
)
|
||||
|
||||
|
||||
def run_performance_for_models(
|
||||
models: List[ModelLaunchSettings],
|
||||
profile_dir: str,
|
||||
test_name: str,
|
||||
base_url: Optional[str] = None,
|
||||
batch_sizes: List[int] = None,
|
||||
input_lens: Tuple[int, ...] = (4096,),
|
||||
output_lens: Tuple[int, ...] = (512,),
|
||||
is_vlm: bool = False,
|
||||
dataset_name: str = "mmmu",
|
||||
) -> dict:
|
||||
"""Run performance tests for multiple models.
|
||||
|
||||
Args:
|
||||
models: List of ModelLaunchSettings to test
|
||||
profile_dir: Directory for performance profiles
|
||||
test_name: Name for the test (used in reports)
|
||||
base_url: Server base URL (default: DEFAULT_URL_FOR_TEST)
|
||||
batch_sizes: Batch sizes for perf test
|
||||
input_lens: Input lengths
|
||||
output_lens: Output lengths
|
||||
is_vlm: Whether these are VLM models
|
||||
dataset_name: Dataset name for VLM benchmarks
|
||||
|
||||
Returns:
|
||||
dict with results:
|
||||
{
|
||||
"all_passed": bool,
|
||||
"results": [PerformanceTestResult, ...]
|
||||
}
|
||||
"""
|
||||
base_url = base_url or DEFAULT_URL_FOR_TEST
|
||||
|
||||
# Setup performance runner
|
||||
perf_runner = NightlyBenchmarkRunner(
|
||||
profile_dir=profile_dir,
|
||||
test_name=test_name,
|
||||
base_url=base_url,
|
||||
)
|
||||
perf_runner.setup_profile_directory()
|
||||
|
||||
all_results = []
|
||||
all_passed = True
|
||||
|
||||
for model in models:
|
||||
print("\n" + "=" * 80)
|
||||
print(f"PERFORMANCE TEST: {model.model_path}")
|
||||
print(f" TP Size: {model.tp_size}")
|
||||
print(f" Extra Args: {model.extra_args}")
|
||||
print("=" * 80)
|
||||
|
||||
result = run_performance_test(
|
||||
model=model,
|
||||
perf_runner=perf_runner,
|
||||
batch_sizes=batch_sizes,
|
||||
input_lens=input_lens,
|
||||
output_lens=output_lens,
|
||||
is_vlm=is_vlm,
|
||||
dataset_name=dataset_name,
|
||||
)
|
||||
|
||||
all_results.append(result)
|
||||
|
||||
if not result.passed:
|
||||
all_passed = False
|
||||
|
||||
# Write performance report
|
||||
perf_runner.write_final_report()
|
||||
|
||||
# Print summary
|
||||
print("\n" + "=" * 60)
|
||||
print(f"Performance Test Summary: {test_name}")
|
||||
print("=" * 60)
|
||||
for result in all_results:
|
||||
status = "PASS" if result.passed else "FAIL"
|
||||
throughput_str = (
|
||||
f", output: {result.output_throughput:.1f} tok/s"
|
||||
if result.output_throughput
|
||||
else ""
|
||||
)
|
||||
print(f" {result.model}: {status}{throughput_str}")
|
||||
if result.error:
|
||||
print(f" Error: {result.error}")
|
||||
|
||||
print("\n" + "=" * 60)
|
||||
print(f"OVERALL: {'ALL PASSED' if all_passed else 'SOME FAILED'}")
|
||||
print("=" * 60 + "\n")
|
||||
|
||||
return {
|
||||
"all_passed": all_passed,
|
||||
"results": all_results,
|
||||
}
|
||||
191
python/sglang/test/run_combined_tests.py
Normal file
191
python/sglang/test/run_combined_tests.py
Normal file
@@ -0,0 +1,191 @@
|
||||
from typing import List, Optional
|
||||
|
||||
from sglang.test.accuracy_test_runner import (
|
||||
AccuracyTestParams,
|
||||
AccuracyTestResult,
|
||||
run_accuracy_test,
|
||||
write_accuracy_github_summary,
|
||||
)
|
||||
from sglang.test.nightly_utils import NightlyBenchmarkRunner
|
||||
from sglang.test.performance_test_runner import (
|
||||
PerformanceTestParams,
|
||||
PerformanceTestResult,
|
||||
run_performance_test,
|
||||
)
|
||||
from sglang.test.test_utils import DEFAULT_URL_FOR_TEST, ModelLaunchSettings, is_in_ci
|
||||
|
||||
|
||||
def run_combined_tests(
|
||||
models: List[ModelLaunchSettings],
|
||||
test_name: str = "NightlyTest",
|
||||
base_url: Optional[str] = None,
|
||||
is_vlm: bool = False,
|
||||
accuracy_params: Optional[AccuracyTestParams] = None,
|
||||
performance_params: Optional[PerformanceTestParams] = None,
|
||||
) -> dict:
|
||||
"""Run performance and/or accuracy tests for a list of models.
|
||||
|
||||
Args:
|
||||
models: List of ModelLaunchSettings to test
|
||||
test_name: Name for the test (used in reports)
|
||||
base_url: Server base URL (default: DEFAULT_URL_FOR_TEST)
|
||||
is_vlm: Whether these are VLM models (affects defaults)
|
||||
accuracy_params: Parameters for accuracy tests (None to skip accuracy)
|
||||
performance_params: Parameters for performance tests (None to skip perf)
|
||||
|
||||
Returns:
|
||||
dict with test results:
|
||||
{
|
||||
"all_passed": bool,
|
||||
"results": [
|
||||
{
|
||||
"model": str,
|
||||
"perf_result": PerformanceTestResult/None,
|
||||
"accuracy_result": AccuracyTestResult/None,
|
||||
"errors": list,
|
||||
},
|
||||
...
|
||||
]
|
||||
}
|
||||
"""
|
||||
base_url = base_url or DEFAULT_URL_FOR_TEST
|
||||
run_perf = performance_params is not None
|
||||
run_accuracy = accuracy_params is not None
|
||||
|
||||
# Print test header
|
||||
print("\n" + "=" * 80)
|
||||
print(f"RUNNING: {test_name}")
|
||||
print(f" Models: {len(models)}")
|
||||
if run_accuracy:
|
||||
print(f" Accuracy dataset: {accuracy_params.dataset}")
|
||||
if run_perf:
|
||||
print(f" Performance batches: {performance_params.batch_sizes}")
|
||||
print("=" * 80)
|
||||
|
||||
# Set up performance parameters
|
||||
if run_perf:
|
||||
perf = performance_params
|
||||
profile_dir = perf.profile_dir or (
|
||||
"performance_profiles_vlms"
|
||||
if is_vlm
|
||||
else "performance_profiles_text_models"
|
||||
)
|
||||
|
||||
perf_runner = NightlyBenchmarkRunner(
|
||||
profile_dir=profile_dir,
|
||||
test_name=test_name,
|
||||
base_url=base_url,
|
||||
)
|
||||
perf_runner.setup_profile_directory()
|
||||
else:
|
||||
perf_runner = None
|
||||
|
||||
# Run tests for each model
|
||||
all_results = []
|
||||
all_passed = True
|
||||
|
||||
for model in models:
|
||||
print("\n" + "=" * 80)
|
||||
print(f"TESTING MODEL CONFIG: {model.model_path}")
|
||||
print(f" TP Size: {model.tp_size}")
|
||||
print(f" Extra Args: {model.extra_args}")
|
||||
print("=" * 80)
|
||||
|
||||
model_result = {
|
||||
"model": model.model_path,
|
||||
"perf_result": None,
|
||||
"accuracy_result": None,
|
||||
"errors": [],
|
||||
}
|
||||
|
||||
# Run performance test
|
||||
if run_perf:
|
||||
perf_result: PerformanceTestResult = run_performance_test(
|
||||
model=model,
|
||||
perf_runner=perf_runner,
|
||||
batch_sizes=performance_params.batch_sizes,
|
||||
input_lens=performance_params.input_lens,
|
||||
output_lens=performance_params.output_lens,
|
||||
is_vlm=is_vlm,
|
||||
dataset_name=performance_params.dataset_name,
|
||||
spec_accept_length_threshold=performance_params.spec_accept_length_threshold,
|
||||
)
|
||||
model_result["perf_result"] = perf_result
|
||||
if not perf_result.passed:
|
||||
all_passed = False
|
||||
model_result["errors"].append(perf_result.error)
|
||||
|
||||
# Run accuracy test
|
||||
if run_accuracy:
|
||||
acc_result: AccuracyTestResult = run_accuracy_test(
|
||||
model=model,
|
||||
params=accuracy_params,
|
||||
base_url=base_url,
|
||||
)
|
||||
model_result["accuracy_result"] = acc_result
|
||||
if not acc_result.passed:
|
||||
all_passed = False
|
||||
model_result["errors"].append(acc_result.error)
|
||||
|
||||
all_results.append(model_result)
|
||||
|
||||
# Write performance report if we ran perf tests
|
||||
if run_perf and perf_runner:
|
||||
perf_runner.write_final_report()
|
||||
|
||||
# Write accuracy results to GitHub summary if in CI
|
||||
if run_accuracy and is_in_ci():
|
||||
accuracy_results = [
|
||||
r["accuracy_result"] for r in all_results if r["accuracy_result"]
|
||||
]
|
||||
write_accuracy_github_summary(
|
||||
test_name, accuracy_params.dataset, accuracy_results
|
||||
)
|
||||
|
||||
# Print summary
|
||||
print("\n" + "=" * 60)
|
||||
print(f"{test_name} Results Summary")
|
||||
if run_accuracy:
|
||||
print(f"Dataset: {accuracy_params.dataset}")
|
||||
print(f"Baseline: {accuracy_params.baseline_accuracy}")
|
||||
print("=" * 60)
|
||||
for i, model_result in enumerate(all_results):
|
||||
print(f"\nModel {i + 1}: {model_result['model']}")
|
||||
if run_perf and model_result["perf_result"]:
|
||||
perf = model_result["perf_result"]
|
||||
throughput_str = (
|
||||
f", output: {perf.output_throughput:.1f} tok/s"
|
||||
if perf.output_throughput
|
||||
else ""
|
||||
)
|
||||
accept_str = (
|
||||
f", accept_len: {perf.avg_spec_accept_length:.2f}"
|
||||
if perf.avg_spec_accept_length
|
||||
else ""
|
||||
)
|
||||
print(
|
||||
f" Performance: {'PASS' if perf.passed else 'FAIL'}{throughput_str}{accept_str}"
|
||||
)
|
||||
if run_accuracy and model_result["accuracy_result"]:
|
||||
acc = model_result["accuracy_result"]
|
||||
print(f" Accuracy: {'PASS' if acc.passed else 'FAIL'}")
|
||||
if acc.score is not None:
|
||||
print(f" Score: {acc.score:.3f}")
|
||||
if model_result["errors"]:
|
||||
print(f" Errors: {model_result['errors']}")
|
||||
|
||||
print("\n" + "=" * 60)
|
||||
print(f"OVERALL: {'ALL TESTS PASSED' if all_passed else 'SOME TESTS FAILED'}")
|
||||
print("=" * 60 + "\n")
|
||||
|
||||
# Raise assertion error if any test failed
|
||||
if not all_passed:
|
||||
failed_models = [r["model"] for r in all_results if r["errors"]]
|
||||
raise AssertionError(
|
||||
f"Tests failed for models: {failed_models}. See results above for details."
|
||||
)
|
||||
|
||||
return {
|
||||
"all_passed": all_passed,
|
||||
"results": all_results,
|
||||
}
|
||||
Reference in New Issue
Block a user