chenchenghao/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 11 Packages Projects Releases Wiki Activity

feature: revamp nightly tests with combined runner (#15324)

Browse Source
This commit is contained in:
Douglas Yang
2025-12-20 19:25:22 -08:00
committed by GitHub
parent 9a3bdf2c95
commit 96740d6983
23 changed files with 1332 additions and 1420 deletions
Download Patch File Download Diff File Expand all files Collapse all files

225
.github/workflows/nightly-test-nvidia.yml vendored
View File

@@ -108,6 +108,17 @@ jobs:
run: |
bash scripts/ci/ci_install_dependency.sh
- name: Run common 8-GPU model tests
if: always()
timeout-minutes: 200
env:
TRACE_BASE_URL: https://raw.githubusercontent.com/sglang-bot/sglang-ci-data/main/traces/${{ github.run_id }}
PERFETTO_RELAY_URL: ${{ vars.PERFETTO_RELAY_URL }}
GPU_CONFIG: "8-gpu-h200"
run: |
cd test
python3 run_suite.py --hw cuda --suite nightly-8-gpu-common --nightly --timeout-per-file=12000 --continue-on-error
- name: Run test
timeout-minutes: 30
env:
@@ -116,82 +127,6 @@ jobs:
cd test
python3 run_suite.py --hw cuda --suite nightly-8-gpu-h200 --nightly --continue-on-error
- name: Run MiniMax-M2 nightly performance test
timeout-minutes: 180
env:
TRACE_BASE_URL: https://raw.githubusercontent.com/sglang-bot/sglang-ci-data/main/traces/${{ github.run_id }}
PERFETTO_RELAY_URL: ${{ vars.PERFETTO_RELAY_URL }}
GPU_CONFIG: "8-gpu-h200"
run: |
rm -rf test/performance_profiles_minimax_m2/
cd test
python3 nightly/test_minimax_m2_perf.py
- name: Publish MiniMax-M2 traces to storage repo
env:
GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }}
GITHUB_RUN_ID: ${{ github.run_id }}
GITHUB_RUN_NUMBER: ${{ github.run_number }}
run: |
python3 scripts/ci/publish_traces.py --traces-dir test/performance_profiles_minimax_m2
- name: Run Qwen3-235B nightly performance test
timeout-minutes: 180
env:
TRACE_BASE_URL: https://raw.githubusercontent.com/sglang-bot/sglang-ci-data/main/traces/${{ github.run_id }}
PERFETTO_RELAY_URL: ${{ vars.PERFETTO_RELAY_URL }}
GPU_CONFIG: "8-gpu-h200"
run: |
rm -rf test/performance_profiles_qwen3_235b/
cd test
python3 nightly/test_qwen3_235b_perf.py
- name: Publish Qwen3-235B traces to storage repo
env:
GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }}
GITHUB_RUN_ID: ${{ github.run_id }}
GITHUB_RUN_NUMBER: ${{ github.run_number }}
run: |
python3 scripts/ci/publish_traces.py --traces-dir test/performance_profiles_qwen3_235b
- name: Run Kimi-K2-Thinking nightly performance test
timeout-minutes: 180
env:
TRACE_BASE_URL: https://raw.githubusercontent.com/sglang-bot/sglang-ci-data/main/traces/${{ github.run_id }}
PERFETTO_RELAY_URL: ${{ vars.PERFETTO_RELAY_URL }}
GPU_CONFIG: "8-gpu-h200"
run: |
rm -rf test/performance_profiles_kimi_k2_thinking/
cd test
python3 nightly/test_kimi_k2_thinking_perf.py
- name: Publish Kimi-K2-Thinking traces to storage repo
env:
GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }}
GITHUB_RUN_ID: ${{ github.run_id }}
GITHUB_RUN_NUMBER: ${{ github.run_number }}
run: |
python3 scripts/ci/publish_traces.py --traces-dir test/performance_profiles_kimi_k2_thinking
- name: Run GLM-4.6 nightly performance test
timeout-minutes: 180
env:
TRACE_BASE_URL: https://raw.githubusercontent.com/sglang-bot/sglang-ci-data/main/traces/${{ github.run_id }}
PERFETTO_RELAY_URL: ${{ vars.PERFETTO_RELAY_URL }}
GPU_CONFIG: "8-gpu-h200"
run: |
rm -rf test/performance_profiles_glm_4_6/
cd test
IS_BLACKWELL=1 python3 nightly/test_glm_4_6_perf.py
- name: Publish GLM-4.6 traces to storage repo
env:
GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }}
GITHUB_RUN_ID: ${{ github.run_id }}
GITHUB_RUN_NUMBER: ${{ github.run_number }}
run: |
python3 scripts/ci/publish_traces.py --traces-dir test/performance_profiles_glm_4_6
# General tests - 8 GPU H20
nightly-test-general-8-gpu-h20:
if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-general-8-gpu-h20')
@@ -446,147 +381,17 @@ jobs:
run: |
IS_BLACKWELL=1 bash scripts/ci/ci_install_dependency.sh
- name: Run Mistral-Large-3 nightly performance test
timeout-minutes: 180
- name: Run common 8-GPU model tests
if: always()
timeout-minutes: 200
env:
TRACE_BASE_URL: https://raw.githubusercontent.com/sglang-bot/sglang-ci-data/main/traces/${{ github.run_id }}
PERFETTO_RELAY_URL: ${{ vars.PERFETTO_RELAY_URL }}
GPU_CONFIG: "8-gpu-b200"
SGLANG_ENABLE_JIT_DEEPGEMM: "0"
run: |
rm -rf test/performance_profiles_mistral_large3/
rm -rf test/performance_profiles_mistral_large3_eagle/
cd test
IS_BLACKWELL=1 python3 nightly/test_mistral_large3_perf.py
- name: Publish Mistral-Large-3 traces to storage repo
env:
GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }}
GITHUB_RUN_ID: ${{ github.run_id }}
GITHUB_RUN_NUMBER: ${{ github.run_number }}
run: |
python3 scripts/ci/publish_traces.py --traces-dir test/performance_profiles_mistral_large3
python3 scripts/ci/publish_traces.py --traces-dir test/performance_profiles_mistral_large3_eagle
- name: Run DeepSeek v3.1 nightly performance test
if: always()
timeout-minutes: 180
env:
TRACE_BASE_URL: https://raw.githubusercontent.com/sglang-bot/sglang-ci-data/main/traces/${{ github.run_id }}
PERFETTO_RELAY_URL: ${{ vars.PERFETTO_RELAY_URL }}
GPU_CONFIG: "8-gpu-b200"
run: |
rm -rf test/performance_profiles_deepseek_v31/
cd test
IS_BLACKWELL=1 python3 nightly/test_deepseek_v31_perf.py
- name: Publish DeepSeek v3.1 traces to storage repo
env:
GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }}
GITHUB_RUN_ID: ${{ github.run_id }}
GITHUB_RUN_NUMBER: ${{ github.run_number }}
run: |
python3 scripts/ci/publish_traces.py --traces-dir test/performance_profiles_deepseek_v31
- name: Run DeepSeek v3.2 nightly performance test
if: always()
timeout-minutes: 180
env:
TRACE_BASE_URL: https://raw.githubusercontent.com/sglang-bot/sglang-ci-data/main/traces/${{ github.run_id }}
PERFETTO_RELAY_URL: ${{ vars.PERFETTO_RELAY_URL }}
GPU_CONFIG: "8-gpu-b200"
run: |
rm -rf test/performance_profiles_deepseek_v32/
cd test
IS_BLACKWELL=1 python3 nightly/test_deepseek_v32_perf.py
- name: Publish DeepSeek v3.2 traces to storage repo
env:
GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }}
GITHUB_RUN_ID: ${{ github.run_id }}
GITHUB_RUN_NUMBER: ${{ github.run_number }}
run: |
python3 scripts/ci/publish_traces.py --traces-dir test/performance_profiles_deepseek_v32
- name: Run Kimi-K2-Thinking nightly performance test
if: always()
timeout-minutes: 180
env:
TRACE_BASE_URL: https://raw.githubusercontent.com/sglang-bot/sglang-ci-data/main/traces/${{ github.run_id }}
PERFETTO_RELAY_URL: ${{ vars.PERFETTO_RELAY_URL }}
GPU_CONFIG: "8-gpu-b200"
run: |
rm -rf test/performance_profiles_kimi_k2_thinking/
cd test
IS_BLACKWELL=1 python3 nightly/test_kimi_k2_thinking_perf.py
- name: Publish Kimi-K2-Thinking traces to storage repo
env:
GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }}
GITHUB_RUN_ID: ${{ github.run_id }}
GITHUB_RUN_NUMBER: ${{ github.run_number }}
run: |
python3 scripts/ci/publish_traces.py --traces-dir test/performance_profiles_kimi_k2_thinking
- name: Run Qwen3-235B nightly performance test
if: always()
timeout-minutes: 180
env:
TRACE_BASE_URL: https://raw.githubusercontent.com/sglang-bot/sglang-ci-data/main/traces/${{ github.run_id }}
PERFETTO_RELAY_URL: ${{ vars.PERFETTO_RELAY_URL }}
GPU_CONFIG: "8-gpu-b200"
run: |
rm -rf test/performance_profiles_qwen3_235b/
cd test
IS_BLACKWELL=1 python3 nightly/test_qwen3_235b_perf.py
- name: Publish Qwen3-235B traces to storage repo
env:
GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }}
GITHUB_RUN_ID: ${{ github.run_id }}
GITHUB_RUN_NUMBER: ${{ github.run_number }}
run: |
python3 scripts/ci/publish_traces.py --traces-dir test/performance_profiles_qwen3_235b
- name: Run GLM-4.6 nightly performance test
if: always()
timeout-minutes: 180
env:
TRACE_BASE_URL: https://raw.githubusercontent.com/sglang-bot/sglang-ci-data/main/traces/${{ github.run_id }}
PERFETTO_RELAY_URL: ${{ vars.PERFETTO_RELAY_URL }}
GPU_CONFIG: "8-gpu-b200"
run: |
rm -rf test/performance_profiles_glm_4_6/
cd test
IS_BLACKWELL=1 python3 nightly/test_glm_4_6_perf.py
- name: Publish GLM-4.6 traces to storage repo
env:
GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }}
GITHUB_RUN_ID: ${{ github.run_id }}
GITHUB_RUN_NUMBER: ${{ github.run_number }}
run: |
python3 scripts/ci/publish_traces.py --traces-dir test/performance_profiles_glm_4_6
- name: Run MiniMax-M2 nightly performance test
if: always()
timeout-minutes: 180
env:
TRACE_BASE_URL: https://raw.githubusercontent.com/sglang-bot/sglang-ci-data/main/traces/${{ github.run_id }}
PERFETTO_RELAY_URL: ${{ vars.PERFETTO_RELAY_URL }}
GPU_CONFIG: "8-gpu-b200"
run: |
rm -rf test/performance_profiles_minimax_m2/
cd test
IS_BLACKWELL=1 python3 nightly/test_minimax_m2_perf.py
- name: Publish MiniMax-M2 traces to storage repo
env:
GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }}
GITHUB_RUN_ID: ${{ github.run_id }}
GITHUB_RUN_NUMBER: ${{ github.run_number }}
run: |
python3 scripts/ci/publish_traces.py --traces-dir test/performance_profiles_minimax_m2
IS_BLACKWELL=1 python3 run_suite.py --hw cuda --suite nightly-8-gpu-common --nightly --timeout-per-file=12000 --continue-on-error
# Final check job
check-all-jobs:

263
test/nightly/accuracy_test_runner.py Normal file
View File

@@ -0,0 +1,263 @@
from dataclasses import dataclass
from types import SimpleNamespace
from typing import List, Optional, Tuple
from sglang.srt.utils import kill_process_tree
from sglang.test.run_eval import run_eval
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
ModelLaunchSettings,
popen_launch_server,
write_github_step_summary,
)
@dataclass
class AccuracyTestParams:
"""Parameters for accuracy testing."""
dataset: str # e.g., "mgsm_en", "gsm8k", "mmmu", "gpqa"
baseline_accuracy: float # Required: minimum accuracy threshold
num_examples: Optional[int] = None
num_threads: Optional[int] = None
max_tokens: Optional[int] = None
return_latency: bool = False
# Extended parameters for special evaluations (e.g., GPQA with thinking mode)
thinking_mode: Optional[str] = None # e.g., "deepseek-v3"
temperature: Optional[float] = None
repeat: Optional[int] = None
@dataclass
class AccuracyTestResult:
"""Result of an accuracy test."""
model: str
dataset: str
passed: bool
score: Optional[float]
baseline_accuracy: float
error: Optional[str]
latency: Optional[float] = None
def write_accuracy_github_summary(
test_name: str,
dataset: str,
results: List[AccuracyTestResult],
) -> None:
"""Write accuracy test results to GitHub step summary.
Args:
test_name: Name of the test
dataset: Dataset name used for evaluation
results: List of AccuracyTestResult objects
"""
summary = f"## {test_name} - Accuracy ({dataset})\n"
summary += "| model | status | score | baseline | error |\n"
summary += "| ----- | ------ | ----- | -------- | ----- |\n"
for result in results:
status_emoji = "✅" if result.passed else "❌"
score_str = f"{result.score:.4f}" if result.score is not None else "N/A"
baseline_str = f"{result.baseline_accuracy:.4f}"
error_str = result.error if result.error else "-"
summary += f"| {result.model} | {status_emoji} | {score_str} | {baseline_str} | {error_str} |\n"
write_github_step_summary(summary)
def _run_simple_eval(
model: ModelLaunchSettings,
base_url: str,
dataset: str,
num_examples: Optional[int] = None,
num_threads: Optional[int] = None,
max_tokens: Optional[int] = None,
return_latency: bool = False,
thinking_mode: Optional[str] = None,
temperature: Optional[float] = None,
repeat: Optional[int] = None,
) -> Tuple[bool, Optional[str], Optional[dict]]:
"""Run evaluation using simple_eval backend (run_eval.py).
Returns:
Tuple of (success, error_message, metrics_dict)
"""
process = None
try:
process = popen_launch_server(
model.model_path,
base_url,
other_args=model.extra_args,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
)
args = SimpleNamespace(
base_url=base_url,
model=model.model_path,
eval_name=dataset,
num_examples=num_examples,
num_threads=num_threads or 1024,
)
if max_tokens is not None:
args.max_tokens = max_tokens
if return_latency:
args.return_latency = True
if thinking_mode is not None:
args.thinking_mode = thinking_mode
if temperature is not None:
args.temperature = temperature
if repeat is not None:
args.repeat = repeat
result = run_eval(args)
# Handle result format (run_eval can return metrics or (metrics, latency))
if return_latency and isinstance(result, tuple):
metrics, latency = result
metrics["latency"] = round(latency, 4)
else:
metrics = result
return True, None, metrics
except Exception as e:
return False, f"Accuracy test exception: {str(e)}", None
finally:
if process:
kill_process_tree(process.pid)
def _run_few_shot_eval(
model: ModelLaunchSettings,
base_url: str,
num_questions: Optional[int] = None,
num_shots: int = 8,
max_tokens: int = 512,
) -> Tuple[bool, Optional[str], Optional[dict]]:
"""Run evaluation using few_shot backend (few_shot_gsm8k.py).
Returns:
Tuple of (success, error_message, metrics_dict)
"""
from sglang.test.few_shot_gsm8k import run_eval as run_few_shot_eval
process = None
try:
process = popen_launch_server(
model.model_path,
base_url,
other_args=model.extra_args,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
)
args = SimpleNamespace(
num_shots=num_shots,
data_path=None,
num_questions=num_questions or 200,
max_new_tokens=max_tokens,
parallel=128,
host="http://127.0.0.1",
port=int(base_url.split(":")[-1]),
)
metrics = run_few_shot_eval(args)
# Normalize metrics format (few_shot returns "accuracy", simple_eval returns "score")
if "accuracy" in metrics and "score" not in metrics:
metrics["score"] = metrics["accuracy"]
return True, None, metrics
except Exception as e:
return False, f"Few-shot evaluation exception: {str(e)}", None
finally:
if process:
kill_process_tree(process.pid)
def run_accuracy_test(
model: ModelLaunchSettings,
params: AccuracyTestParams,
base_url: Optional[str] = None,
) -> AccuracyTestResult:
"""Run accuracy test for a single model.
Args:
model: ModelLaunchSettings with model config
params: AccuracyTestParams with dataset, baseline, and optional settings
base_url: Server base URL (default: DEFAULT_URL_FOR_TEST)
Returns:
AccuracyTestResult with test outcome
"""
base_url = base_url or DEFAULT_URL_FOR_TEST
print(f"\n{'='*60}")
print(f"Running ACCURACY test for {model.model_path}")
print(f" Dataset: {params.dataset}")
print(f" Baseline: {params.baseline_accuracy}")
print(f"{'='*60}\n")
# Run evaluation based on dataset type
if params.dataset == "gsm8k":
success, error, metrics = _run_few_shot_eval(
model=model,
base_url=base_url,
num_questions=params.num_examples,
max_tokens=params.max_tokens or 512,
)
else:
success, error, metrics = _run_simple_eval(
model=model,
base_url=base_url,
dataset=params.dataset,
num_examples=params.num_examples,
num_threads=params.num_threads,
max_tokens=params.max_tokens,
return_latency=params.return_latency,
thinking_mode=params.thinking_mode,
temperature=params.temperature,
repeat=params.repeat,
)
if not success:
print(f"✗ Accuracy test failed for {model.model_path}: {error}")
return AccuracyTestResult(
model=model.model_path,
dataset=params.dataset,
passed=False,
score=None,
baseline_accuracy=params.baseline_accuracy,
error=error,
)
# Validate against baseline
score = metrics.get("score", 0.0)
passed = score >= params.baseline_accuracy
latency = metrics.get("latency")
if passed:
print(f"✓ Accuracy {score:.3f} >= baseline {params.baseline_accuracy:.3f}")
else:
error = f"Accuracy {score:.3f} below baseline {params.baseline_accuracy:.3f}"
print(f"✗ {error}")
return AccuracyTestResult(
model=model.model_path,
dataset=params.dataset,
passed=passed,
score=score,
baseline_accuracy=params.baseline_accuracy,
error=error if not passed else None,
latency=latency,
)

35
test/nightly/nightly_utils.py
View File

@@ -6,6 +6,8 @@ import subprocess
import time
from typing import List, Optional, Tuple
import requests
from sglang.srt.utils import kill_process_tree
from sglang.test.nightly_bench_utils import BenchmarkResult, generate_markdown_report
from sglang.test.test_utils import (
@@ -211,7 +213,7 @@ class NightlyBenchmarkRunner:
other_args: Optional[List[str]] = None,
variant: str = "",
extra_bench_args: Optional[List[str]] = None,
) -> Tuple[List[BenchmarkResult], bool]:
) -> Tuple[List[BenchmarkResult], bool, Optional[float]]:
"""Run a complete benchmark for a single model with server management.
This method handles:
@@ -219,6 +221,7 @@ class NightlyBenchmarkRunner:
- Profile filename generation
- Benchmark command construction and execution
- Result loading and parsing
- Fetching speculative decoding accept length (for MTP/EAGLE)
Args:
model_path: Path to the model
@@ -230,9 +233,10 @@ class NightlyBenchmarkRunner:
extra_bench_args: Extra arguments for the benchmark command
Returns:
Tuple of (list of BenchmarkResult objects, success_bool)
Tuple of (list of BenchmarkResult objects, success_bool, avg_spec_accept_length or None)
"""
benchmark_results = []
avg_spec_accept_length = None
model_description = f"{model_path}" + (f" ({variant})" if variant else "")
# Launch server
@@ -268,19 +272,42 @@ class NightlyBenchmarkRunner:
result, cmd_success = self.run_benchmark_command(command, model_description)
if not cmd_success:
return benchmark_results, False
return benchmark_results, False, None
# Load results
benchmark_results, load_success = self.load_benchmark_results(
json_output_file, model_description
)
return benchmark_results, load_success
# Fetch speculative decoding accept length before killing server
avg_spec_accept_length = self._get_spec_accept_length()
return benchmark_results, load_success, avg_spec_accept_length
finally:
# Always clean up server process
kill_process_tree(process.pid)
def _get_spec_accept_length(self) -> Optional[float]:
"""Query the server for avg_spec_accept_length metric.
Returns:
The average speculative decoding accept length, or None if not available.
"""
try:
response = requests.get(f"{self.base_url}/get_server_info", timeout=10)
if response.status_code == 200:
server_info = response.json()
internal_states = server_info.get("internal_states", [])
if internal_states and len(internal_states) > 0:
accept_length = internal_states[0].get("avg_spec_accept_length")
if accept_length is not None:
print(f" avg_spec_accept_length={accept_length:.2f}")
return accept_length
except Exception as e:
print(f" Warning: Could not fetch spec accept length: {e}")
return None
def add_report(self, results: List[BenchmarkResult]) -> None:
"""Add benchmark results to the full report.

231
test/nightly/performance_test_runner.py Normal file
View File

@@ -0,0 +1,231 @@
from dataclasses import dataclass, field
from typing import List, Optional, Tuple
from nightly_utils import NightlyBenchmarkRunner
from sglang.test.nightly_bench_utils import BenchmarkResult
from sglang.test.test_utils import DEFAULT_URL_FOR_TEST, ModelLaunchSettings
@dataclass
class PerformanceTestParams:
"""Parameters for performance testing."""
batch_sizes: List[int] = field(default_factory=lambda: [1, 8, 16, 64])
input_lens: Tuple[int, ...] = (4096,)
output_lens: Tuple[int, ...] = (512,)
profile_dir: Optional[str] = None # None = auto-generate based on is_vlm
dataset_name: str = "mmmu" # For VLM perf test
# MTP/EAGLE speculative decoding: minimum accept length threshold (None = no validation)
spec_accept_length_threshold: Optional[float] = None
@dataclass
class PerformanceTestResult:
"""Result of a performance test.
Aggregates metrics across all batch sizes tested for a single model.
"""
model: str
passed: bool
error: Optional[str]
# Aggregate metrics (from the largest batch size result, or None if failed)
latency: Optional[float] = None
input_throughput: Optional[float] = None
output_throughput: Optional[float] = None
overall_throughput: Optional[float] = None
# All individual benchmark results
benchmark_results: Optional[List[BenchmarkResult]] = None
# MTP/EAGLE speculative decoding metric
avg_spec_accept_length: Optional[float] = None
def run_performance_test(
model: ModelLaunchSettings,
perf_runner: NightlyBenchmarkRunner,
batch_sizes: List[int] = None,
input_lens: Tuple[int, ...] = (4096,),
output_lens: Tuple[int, ...] = (512,),
is_vlm: bool = False,
dataset_name: str = "mmmu",
spec_accept_length_threshold: Optional[float] = None,
) -> PerformanceTestResult:
# Set default for mutable argument
if batch_sizes is None:
batch_sizes = [1, 8, 16, 64]
print(f"\n{'='*60}")
print(f"Running PERFORMANCE test for {model.model_path}")
print(f" Batch sizes: {batch_sizes}")
print(f" Input lens: {input_lens}")
print(f" Output lens: {output_lens}")
if spec_accept_length_threshold is not None:
print(f" Spec accept length threshold: {spec_accept_length_threshold}")
print(f"{'='*60}\n")
# Build extra args for benchmarks
extra_bench_args = ["--trust-remote-code"]
if is_vlm:
extra_bench_args.append(f"--dataset-name={dataset_name}")
try:
results, success, avg_spec_accept_length = perf_runner.run_benchmark_for_model(
model_path=model.model_path,
batch_sizes=batch_sizes,
input_lens=input_lens,
output_lens=output_lens,
other_args=model.extra_args,
extra_bench_args=extra_bench_args,
)
if success and results:
perf_runner.add_report(results)
print(f"✓ Performance test succeeded for {model.model_path}")
# Validate speculative decoding accept length if threshold is set
error_msg = None
passed = True
if spec_accept_length_threshold is not None:
if avg_spec_accept_length is None:
error_msg = f"Spec accept length threshold set but no accept length reported"
passed = False
print(f"✗ {error_msg}")
elif avg_spec_accept_length < spec_accept_length_threshold:
error_msg = (
f"Spec accept length {avg_spec_accept_length:.2f} < "
f"threshold {spec_accept_length_threshold}"
)
passed = False
print(f"✗ {error_msg}")
else:
print(
f"✓ Spec accept length {avg_spec_accept_length:.2f} >= "
f"threshold {spec_accept_length_threshold}"
)
# Extract aggregate metrics from the largest batch size result
largest_batch_result = max(results, key=lambda r: r.batch_size)
return PerformanceTestResult(
model=model.model_path,
passed=passed,
error=error_msg,
latency=largest_batch_result.latency,
input_throughput=largest_batch_result.input_throughput,
output_throughput=largest_batch_result.output_throughput,
overall_throughput=largest_batch_result.overall_throughput,
benchmark_results=results,
avg_spec_accept_length=avg_spec_accept_length,
)
else:
error_msg = f"Performance test failed for {model.model_path}"
print(f"✗ {error_msg}")
return PerformanceTestResult(
model=model.model_path,
passed=False,
error=error_msg,
)
except Exception as e:
error_msg = f"Performance test exception for {model.model_path}: {str(e)}"
print(f"✗ {error_msg}")
return PerformanceTestResult(
model=model.model_path,
passed=False,
error=error_msg,
)
def run_performance_for_models(
models: List[ModelLaunchSettings],
profile_dir: str,
test_name: str,
base_url: Optional[str] = None,
batch_sizes: List[int] = None,
input_lens: Tuple[int, ...] = (4096,),
output_lens: Tuple[int, ...] = (512,),
is_vlm: bool = False,
dataset_name: str = "mmmu",
) -> dict:
"""Run performance tests for multiple models.
Args:
models: List of ModelLaunchSettings to test
profile_dir: Directory for performance profiles
test_name: Name for the test (used in reports)
base_url: Server base URL (default: DEFAULT_URL_FOR_TEST)
batch_sizes: Batch sizes for perf test
input_lens: Input lengths
output_lens: Output lengths
is_vlm: Whether these are VLM models
dataset_name: Dataset name for VLM benchmarks
Returns:
dict with results:
{
"all_passed": bool,
"results": [PerformanceTestResult, ...]
}
"""
base_url = base_url or DEFAULT_URL_FOR_TEST
# Setup performance runner
perf_runner = NightlyBenchmarkRunner(
profile_dir=profile_dir,
test_name=test_name,
base_url=base_url,
)
perf_runner.setup_profile_directory()
all_results = []
all_passed = True
for model in models:
print("\n" + "=" * 80)
print(f"PERFORMANCE TEST: {model.model_path}")
print(f" TP Size: {model.tp_size}")
print(f" Extra Args: {model.extra_args}")
print("=" * 80)
result = run_performance_test(
model=model,
perf_runner=perf_runner,
batch_sizes=batch_sizes,
input_lens=input_lens,
output_lens=output_lens,
is_vlm=is_vlm,
dataset_name=dataset_name,
)
all_results.append(result)
if not result.passed:
all_passed = False
# Write performance report
perf_runner.write_final_report()
# Print summary
print("\n" + "=" * 60)
print(f"Performance Test Summary: {test_name}")
print("=" * 60)
for result in all_results:
status = "PASS" if result.passed else "FAIL"
throughput_str = (
f", output: {result.output_throughput:.1f} tok/s"
if result.output_throughput
else ""
)
print(f" {result.model}: {status}{throughput_str}")
if result.error:
print(f" Error: {result.error}")
print("\n" + "=" * 60)
print(f"OVERALL: {'ALL PASSED' if all_passed else 'SOME FAILED'}")
print("=" * 60 + "\n")
return {
"all_passed": all_passed,
"results": all_results,
}

192
test/nightly/run_combined_tests.py Normal file
View File

@@ -0,0 +1,192 @@
from typing import List, Optional
from accuracy_test_runner import (
AccuracyTestParams,
AccuracyTestResult,
run_accuracy_test,
write_accuracy_github_summary,
)
from nightly_utils import NightlyBenchmarkRunner
from performance_test_runner import (
PerformanceTestParams,
PerformanceTestResult,
run_performance_test,
)
from sglang.test.test_utils import DEFAULT_URL_FOR_TEST, ModelLaunchSettings, is_in_ci
def run_combined_tests(
models: List[ModelLaunchSettings],
test_name: str = "NightlyTest",
base_url: Optional[str] = None,
is_vlm: bool = False,
accuracy_params: Optional[AccuracyTestParams] = None,
performance_params: Optional[PerformanceTestParams] = None,
) -> dict:
"""Run performance and/or accuracy tests for a list of models.
Args:
models: List of ModelLaunchSettings to test
test_name: Name for the test (used in reports)
base_url: Server base URL (default: DEFAULT_URL_FOR_TEST)
is_vlm: Whether these are VLM models (affects defaults)
accuracy_params: Parameters for accuracy tests (None to skip accuracy)
performance_params: Parameters for performance tests (None to skip perf)
Returns:
dict with test results:
{
"all_passed": bool,
"results": [
{
"model": str,
"perf_result": PerformanceTestResult/None,
"accuracy_result": AccuracyTestResult/None,
"errors": list,
},
...
]
}
"""
base_url = base_url or DEFAULT_URL_FOR_TEST
run_perf = performance_params is not None
run_accuracy = accuracy_params is not None
# Print test header
print("\n" + "=" * 80)
print(f"RUNNING: {test_name}")
print(f" Models: {len(models)}")
if run_accuracy:
print(f" Accuracy dataset: {accuracy_params.dataset}")
if run_perf:
print(f" Performance batches: {performance_params.batch_sizes}")
print("=" * 80)
# Set up performance parameters
if run_perf:
perf = performance_params
profile_dir = perf.profile_dir or (
"performance_profiles_vlms"
if is_vlm
else "performance_profiles_text_models"
)
perf_runner = NightlyBenchmarkRunner(
profile_dir=profile_dir,
test_name=test_name,
base_url=base_url,
)
perf_runner.setup_profile_directory()
else:
perf_runner = None
# Run tests for each model
all_results = []
all_passed = True
for model in models:
print("\n" + "=" * 80)
print(f"TESTING MODEL CONFIG: {model.model_path}")
print(f" TP Size: {model.tp_size}")
print(f" Extra Args: {model.extra_args}")
print("=" * 80)
model_result = {
"model": model.model_path,
"perf_result": None,
"accuracy_result": None,
"errors": [],
}
# Run performance test
if run_perf:
perf_result: PerformanceTestResult = run_performance_test(
model=model,
perf_runner=perf_runner,
batch_sizes=performance_params.batch_sizes,
input_lens=performance_params.input_lens,
output_lens=performance_params.output_lens,
is_vlm=is_vlm,
dataset_name=performance_params.dataset_name,
spec_accept_length_threshold=performance_params.spec_accept_length_threshold,
)
model_result["perf_result"] = perf_result
if not perf_result.passed:
all_passed = False
model_result["errors"].append(perf_result.error)
# Run accuracy test
if run_accuracy:
acc_result: AccuracyTestResult = run_accuracy_test(
model=model,
params=accuracy_params,
base_url=base_url,
)
model_result["accuracy_result"] = acc_result
if not acc_result.passed:
all_passed = False
model_result["errors"].append(acc_result.error)
all_results.append(model_result)
# Write performance report if we ran perf tests
if run_perf and perf_runner:
perf_runner.write_final_report()
# Write accuracy results to GitHub summary if in CI
if run_accuracy and is_in_ci():
accuracy_results = [
r["accuracy_result"] for r in all_results if r["accuracy_result"]
]
write_accuracy_github_summary(
test_name, accuracy_params.dataset, accuracy_results
)
# Print summary
print("\n" + "=" * 60)
print(f"{test_name} Results Summary")
if run_accuracy:
print(f"Dataset: {accuracy_params.dataset}")
print(f"Baseline: {accuracy_params.baseline_accuracy}")
print("=" * 60)
for i, model_result in enumerate(all_results):
print(f"\nModel {i + 1}: {model_result['model']}")
if run_perf and model_result["perf_result"]:
perf = model_result["perf_result"]
throughput_str = (
f", output: {perf.output_throughput:.1f} tok/s"
if perf.output_throughput
else ""
)
accept_str = (
f", accept_len: {perf.avg_spec_accept_length:.2f}"
if perf.avg_spec_accept_length
else ""
)
print(
f" Performance: {'PASS' if perf.passed else 'FAIL'}{throughput_str}{accept_str}"
)
if run_accuracy and model_result["accuracy_result"]:
acc = model_result["accuracy_result"]
print(f" Accuracy: {'PASS' if acc.passed else 'FAIL'}")
if acc.score is not None:
print(f" Score: {acc.score:.3f}")
if model_result["errors"]:
print(f" Errors: {model_result['errors']}")
print("\n" + "=" * 60)
print(f"OVERALL: {'ALL TESTS PASSED' if all_passed else 'SOME TESTS FAILED'}")
print("=" * 60 + "\n")
# Raise assertion error if any test failed
if not all_passed:
failed_models = [r["model"] for r in all_results if r["errors"]]
raise AssertionError(
f"Tests failed for models: {failed_models}. See results above for details."
)
return {
"all_passed": all_passed,
"results": all_results,
}

87
test/nightly/test_deepseek_v31_perf.py
View File

@@ -1,87 +0,0 @@
import unittest
from nightly_utils import NightlyBenchmarkRunner
from sglang.test.test_utils import DEFAULT_URL_FOR_TEST, _parse_int_list_env
DEEPSEEK_V31_MODEL_PATH = "deepseek-ai/DeepSeek-V3.1"
PROFILE_DIR = "performance_profiles_deepseek_v31"
class TestNightlyDeepseekV31Performance(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.model = DEEPSEEK_V31_MODEL_PATH
cls.base_url = DEFAULT_URL_FOR_TEST
cls.batch_sizes = [1, 1, 8, 16, 64]
cls.input_lens = tuple(_parse_int_list_env("NIGHTLY_INPUT_LENS", "4096"))
cls.output_lens = tuple(_parse_int_list_env("NIGHTLY_OUTPUT_LENS", "512"))
# Define variant configurations
cls.variants = [
{
"name": "basic",
"other_args": [
"--trust-remote-code",
"--tp",
"8",
"--model-loader-extra-config",
'{"enable_multithread_load": true}',
],
},
{
"name": "mtp",
"other_args": [
"--trust-remote-code",
"--tp",
"8",
"--speculative-algorithm",
"EAGLE",
"--speculative-num-steps",
"3",
"--speculative-eagle-topk",
"1",
"--speculative-num-draft-tokens",
"4",
"--mem-frac",
"0.7",
"--model-loader-extra-config",
'{"enable_multithread_load": true}',
],
},
]
cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url)
cls.runner.setup_profile_directory()
def test_bench_one_batch(self):
failed_variants = []
try:
for variant_config in self.variants:
with self.subTest(variant=variant_config["name"]):
results, success = self.runner.run_benchmark_for_model(
model_path=self.model,
batch_sizes=self.batch_sizes,
input_lens=self.input_lens,
output_lens=self.output_lens,
other_args=variant_config["other_args"],
variant=variant_config["name"],
)
if not success:
failed_variants.append(variant_config["name"])
self.runner.add_report(results)
finally:
self.runner.write_final_report()
if failed_variants:
raise AssertionError(
f"Benchmark failed for {self.model} with the following variants: "
f"{', '.join(failed_variants)}"
)
if __name__ == "__main__":
unittest.main()

102
test/nightly/test_deepseek_v32_gpqa.py
View File

@@ -1,102 +0,0 @@
import unittest
from types import SimpleNamespace
from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k
from sglang.test.run_eval import run_eval
from sglang.test.test_utils import (
DEFAULT_URL_FOR_TEST,
CustomTestCase,
is_in_ci,
popen_launch_server,
try_cached_model,
write_github_step_summary,
)
register_cuda_ci(est_time=3600, suite="nightly-8-gpu-b200", nightly=True)
# Use the latest version of DeepSeek-V3.2
DEEPSEEK_V32_MODEL_PATH = "deepseek-ai/DeepSeek-V3.2"
SERVER_LAUNCH_TIMEOUT = 1200
class TestDeepseekV32Accuracy(CustomTestCase):
@classmethod
def setUpClass(cls):
cls.model = try_cached_model(DEEPSEEK_V32_MODEL_PATH)
cls.base_url = DEFAULT_URL_FOR_TEST
other_args = [
"--trust-remote-code",
"--tp",
"8",
"--enable-dp-attention",
"--dp",
"8",
"--tool-call-parser",
"deepseekv32",
"--reasoning-parser",
"deepseek-v3",
"--model-loader-extra-config",
'{"enable_multithread_load": true,"num_threads": 64}',
]
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=SERVER_LAUNCH_TIMEOUT,
other_args=other_args,
)
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
def test_a_gsm8k(
self,
):
args = SimpleNamespace(
num_shots=20,
data_path=None,
num_questions=1400,
parallel=1400,
max_new_tokens=512,
host="http://127.0.0.1",
port=int(self.base_url.split(":")[-1]),
)
metrics = run_eval_few_shot_gsm8k(args)
print(f"{metrics=}")
if is_in_ci():
write_github_step_summary(
f"### test_gsm8k (deepseek-v32)\n" f'{metrics["accuracy"]=:.3f}\n'
)
self.assertGreater(metrics["accuracy"], 0.935)
def test_gpqa(self):
args = SimpleNamespace(
base_url=self.base_url,
model=DEEPSEEK_V32_MODEL_PATH,
eval_name="gpqa",
num_examples=198,
# use enough threads to allow parallelism
num_threads=198,
max_tokens=120000,
thinking_mode="deepseek-v3",
temperature=0.1,
# Repeat 4 times for shorter runtime. Ideally we should repeat at least 8 times.
repeat=4,
)
print(f"Evaluation start for gpqa")
metrics = run_eval(args)
print(f"Evaluation end for gpqa: {metrics=}, expected_score=0.835")
mean_score = metrics["mean_score"]
self.assertGreaterEqual(mean_score, 0.835)
if is_in_ci():
write_github_step_summary(
f"### test_gpqa (deepseek-v32)\n" f"Mean Score: {mean_score:.3f}\n"
)
if __name__ == "__main__":
unittest.main()

224
test/nightly/test_deepseek_v32_nsabackend.py
View File

@@ -1,224 +0,0 @@
import os
import unittest
from types import SimpleNamespace
from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
CustomTestCase,
is_in_ci,
popen_launch_server,
write_github_step_summary,
)
register_cuda_ci(est_time=600, suite="nightly-8-gpu-h200", nightly=True)
DEEPSEEK_V32_MODEL_PATH = "deepseek-ai/DeepSeek-V3.2-Exp"
# Global list to collect results
TEST_RESULTS = []
class TestDeepseekV32NasBackend_flashmla(CustomTestCase):
@classmethod
def setUpClass(cls):
cls.model = DEEPSEEK_V32_MODEL_PATH
cls.base_url = DEFAULT_URL_FOR_TEST
other_args = [
"--trust-remote-code",
"--attention-backend",
"nsa",
"--nsa-prefill-backend",
"flashmla_sparse",
"--nsa-decode-backend",
"flashmla_kv",
"--tp",
"8",
"--dp",
"8",
"--enable-dp-attention",
]
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=other_args,
)
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
def test_a_gsm8k(
self,
): # Append an "a" to make this test run first (alphabetically) to warm up the server
args = SimpleNamespace(
num_shots=20,
data_path=None,
num_questions=1400,
parallel=1400,
max_new_tokens=512,
host="http://127.0.0.1",
port=int(self.base_url.split(":")[-1]),
)
metrics = run_eval_few_shot_gsm8k(args)
print(f"{metrics=}")
if is_in_ci():
TEST_RESULTS.append(
{
"variant": "flashmla",
"prefill_backend": "flashmla_sparse",
"decode_backend": "flashmla_kv",
"kv_cache": "fp16",
"accuracy": metrics["accuracy"],
}
)
self.assertGreater(metrics["accuracy"], 0.935)
class TestDeepseekV32NasBackend_fa3(CustomTestCase):
@classmethod
def setUpClass(cls):
cls.model = DEEPSEEK_V32_MODEL_PATH
cls.base_url = DEFAULT_URL_FOR_TEST
other_args = [
"--trust-remote-code",
"--attention-backend",
"nsa",
"--nsa-prefill-backend",
"fa3",
"--nsa-decode-backend",
"fa3",
"--tp",
"8",
"--dp",
"8",
"--enable-dp-attention",
]
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=other_args,
)
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
def test_a_gsm8k(
self,
): # Append an "a" to make this test run first (alphabetically) to warm up the server
args = SimpleNamespace(
num_shots=20,
data_path=None,
num_questions=1400,
parallel=1400,
max_new_tokens=512,
host="http://127.0.0.1",
port=int(self.base_url.split(":")[-1]),
)
metrics = run_eval_few_shot_gsm8k(args)
print(f"{metrics=}")
if is_in_ci():
TEST_RESULTS.append(
{
"variant": "fa3",
"prefill_backend": "fa3",
"decode_backend": "fa3",
"kv_cache": "fp16",
"accuracy": metrics["accuracy"],
}
)
self.assertGreater(metrics["accuracy"], 0.935)
class TestDeepseekV32NasBackend_fp8kvcache(CustomTestCase):
@classmethod
def setUpClass(cls):
cls.model = DEEPSEEK_V32_MODEL_PATH
cls.base_url = DEFAULT_URL_FOR_TEST
other_args = [
"--trust-remote-code",
"--attention-backend",
"nsa",
"--kv-cache-dtype",
"fp8_e4m3",
"--tp",
"8",
"--dp",
"8",
"--enable-dp-attention",
]
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=other_args,
)
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
def test_a_gsm8k(
self,
): # Append an "a" to make this test run first (alphabetically) to warm up the server
args = SimpleNamespace(
num_shots=20,
data_path=None,
num_questions=1400,
parallel=1400,
max_new_tokens=512,
host="http://127.0.0.1",
port=int(self.base_url.split(":")[-1]),
)
metrics = run_eval_few_shot_gsm8k(args)
print(f"{metrics=}")
if is_in_ci():
TEST_RESULTS.append(
{
"variant": "fp8kvcache",
"prefill_backend": "default",
"decode_backend": "default",
"kv_cache": "fp8_e4m3",
"accuracy": metrics["accuracy"],
}
)
# Write the summary table after all tests complete
_write_summary_table()
self.assertGreater(metrics["accuracy"], 0.935)
def _write_summary_table():
"""Write a markdown table with all test results."""
if not TEST_RESULTS:
return
gpu_config = os.getenv("GPU_CONFIG", "8-gpu-h200")
# Build table header
summary = f"### {DEEPSEEK_V32_MODEL_PATH} GSM8K Accuracy [{gpu_config}]\n\n"
summary += "| Variant | Prefill Backend | Decode Backend | KV Cache | Accuracy |\n"
summary += "|---------|-----------------|----------------|----------|----------|\n"
# Add each result as a row
for result in TEST_RESULTS:
summary += (
f"| {result['variant']} | {result['prefill_backend']} | "
f"{result['decode_backend']} | {result['kv_cache']} | "
f"{result['accuracy']:.3f} |\n"
)
write_github_step_summary(summary)
if __name__ == "__main__":
unittest.main()

123
test/nightly/test_deepseek_v32_perf.py
View File

@@ -1,123 +0,0 @@
import unittest
from nightly_utils import NightlyBenchmarkRunner
from sglang.test.test_utils import DEFAULT_URL_FOR_TEST, _parse_int_list_env
DEEPSEEK_V32_MODEL_PATH = "deepseek-ai/DeepSeek-V3.2-Exp"
PROFILE_DIR = "performance_profiles_deepseek_v32"
class TestNightlyDeepseekV32Performance(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.model = DEEPSEEK_V32_MODEL_PATH
cls.base_url = DEFAULT_URL_FOR_TEST
cls.batch_sizes = [1, 8, 16, 64]
cls.input_lens = tuple(_parse_int_list_env("NIGHTLY_INPUT_LENS", "4096"))
cls.output_lens = tuple(_parse_int_list_env("NIGHTLY_OUTPUT_LENS", "512"))
# Define variant configurations
cls.variants = [
{
"name": "dp",
"other_args": [
"--trust-remote-code",
"--tp",
"8",
"--dp",
"8",
"--enable-dp-attention",
"--model-loader-extra-config",
'{"enable_multithread_load": true}',
],
},
{
"name": "dp+mtp",
"other_args": [
"--trust-remote-code",
"--tp",
"8",
"--dp",
"8",
"--enable-dp-attention",
"--speculative-algorithm",
"EAGLE",
"--speculative-num-steps",
"3",
"--speculative-eagle-topk",
"1",
"--speculative-num-draft-tokens",
"4",
"--mem-frac",
"0.7",
"--model-loader-extra-config",
'{"enable_multithread_load": true}',
],
},
{
"name": "tp",
"other_args": [
"--trust-remote-code",
"--tp",
"8",
"--model-loader-extra-config",
'{"enable_multithread_load": true}',
],
},
{
"name": "tp+mtp",
"other_args": [
"--trust-remote-code",
"--tp",
"8",
"--speculative-algorithm",
"EAGLE",
"--speculative-num-steps",
"3",
"--speculative-eagle-topk",
"1",
"--speculative-num-draft-tokens",
"4",
"--mem-frac",
"0.7",
"--model-loader-extra-config",
'{"enable_multithread_load": true}',
],
},
]
cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url)
cls.runner.setup_profile_directory()
def test_bench_one_batch(self):
failed_variants = []
try:
for variant_config in self.variants:
with self.subTest(variant=variant_config["name"]):
results, success = self.runner.run_benchmark_for_model(
model_path=self.model,
batch_sizes=self.batch_sizes,
input_lens=self.input_lens,
output_lens=self.output_lens,
other_args=variant_config["other_args"],
variant=variant_config["name"],
)
if not success:
failed_variants.append(variant_config["name"])
self.runner.add_report(results)
finally:
self.runner.write_final_report()
if failed_variants:
raise AssertionError(
f"Benchmark failed for {self.model} with the following variants: "
f"{', '.join(failed_variants)}"
)
if __name__ == "__main__":
unittest.main()

263
test/nightly/test_deepseek_v32_tp.py
View File

@@ -1,263 +0,0 @@
import os
import unittest
from types import SimpleNamespace
import requests
from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k
from sglang.test.send_one import BenchArgs, send_one_prompt
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
CustomTestCase,
is_in_ci,
popen_launch_server,
write_github_step_summary,
)
register_cuda_ci(est_time=900, suite="nightly-8-gpu-h200", nightly=True)
DEEPSEEK_V32_MODEL_PATH = "deepseek-ai/DeepSeek-V3.2-Exp"
# Global list to collect results
TEST_RESULTS = []
class TestDeepseekV32_TP(CustomTestCase):
@classmethod
def setUpClass(cls):
cls.model = DEEPSEEK_V32_MODEL_PATH
cls.base_url = DEFAULT_URL_FOR_TEST
# Pure TP configuration without --dp and --enable-dp-attention
other_args = [
"--trust-remote-code",
"--attention-backend",
"nsa",
"--nsa-prefill-backend",
"flashmla_sparse",
"--nsa-decode-backend",
"flashmla_kv",
"--tp",
"8",
]
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=other_args,
)
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
def test_a_gsm8k(
self,
): # Append an "a" to make this test run first (alphabetically) to warm up the server
args = SimpleNamespace(
num_shots=20,
data_path=None,
num_questions=1400,
parallel=1400,
max_new_tokens=512,
host="http://127.0.0.1",
port=int(self.base_url.split(":")[-1]),
)
metrics = run_eval_few_shot_gsm8k(args)
print(f"{metrics=}")
if is_in_ci():
TEST_RESULTS.append(
{
"variant": "pure_tp",
"prefill_backend": "flashmla_sparse",
"decode_backend": "flashmla_kv",
"kv_cache": "fp16",
"accuracy": metrics["accuracy"],
}
)
self.assertGreater(metrics["accuracy"], 0.935)
class TestDeepseekV32_Partial_TP(CustomTestCase):
@classmethod
def setUpClass(cls):
cls.model = DEEPSEEK_V32_MODEL_PATH
cls.base_url = DEFAULT_URL_FOR_TEST
# Partial TP configuration with dp=4 and dp-attention enabled
other_args = [
"--trust-remote-code",
"--attention-backend",
"nsa",
"--nsa-prefill-backend",
"flashmla_sparse",
"--nsa-decode-backend",
"flashmla_kv",
"--tp",
"8",
"--dp",
"4",
"--enable-dp-attention",
]
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=other_args,
)
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
def test_a_gsm8k(
self,
): # Append an "a" to make this test run first (alphabetically) to warm up the server
args = SimpleNamespace(
num_shots=20,
data_path=None,
num_questions=1400,
parallel=1400,
max_new_tokens=512,
host="http://127.0.0.1",
port=int(self.base_url.split(":")[-1]),
)
metrics = run_eval_few_shot_gsm8k(args)
print(f"{metrics=}")
if is_in_ci():
TEST_RESULTS.append(
{
"variant": "partial_tp",
"prefill_backend": "flashmla_sparse",
"decode_backend": "flashmla_kv",
"kv_cache": "fp16",
"accuracy": metrics["accuracy"],
}
)
self.assertGreater(metrics["accuracy"], 0.935)
class TestDeepseekV32_TP_MTP(CustomTestCase):
"""Test DeepSeek V3.2 with pure TP + MTP (EAGLE speculative decoding)."""
@classmethod
def setUpClass(cls):
cls.model = DEEPSEEK_V32_MODEL_PATH
cls.base_url = DEFAULT_URL_FOR_TEST
other_args = [
"--trust-remote-code",
"--tp",
"8",
"--speculative-algorithm",
"EAGLE",
"--speculative-num-steps",
"3",
"--speculative-eagle-topk",
"1",
"--speculative-num-draft-tokens",
"4",
"--mem-frac",
"0.7",
]
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=other_args,
)
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
def test_a_gsm8k(self):
requests.get(self.base_url + "/flush_cache")
args = SimpleNamespace(
num_shots=20,
data_path=None,
num_questions=1400,
parallel=1400,
max_new_tokens=512,
host="http://127.0.0.1",
port=int(self.base_url.split(":")[-1]),
)
metrics = run_eval_few_shot_gsm8k(args)
print(f"{metrics=}")
server_info = requests.get(self.base_url + "/get_server_info")
avg_spec_accept_length = server_info.json()["internal_states"][0][
"avg_spec_accept_length"
]
print(f"{avg_spec_accept_length=}")
if is_in_ci():
TEST_RESULTS.append(
{
"variant": "tp_mtp",
"prefill_backend": "flashmla_sparse",
"decode_backend": "flashmla_kv",
"kv_cache": "fp16",
"accuracy": metrics["accuracy"],
"avg_spec_accept_length": avg_spec_accept_length,
}
)
self.assertGreater(metrics["accuracy"], 0.935)
self.assertGreater(avg_spec_accept_length, 2.5)
def test_bs_1_speed(self):
args = BenchArgs(port=int(self.base_url.split(":")[-1]), max_new_tokens=2048)
acc_length, speed = send_one_prompt(args)
print(f"{acc_length=:.2f} {speed=:.2f}")
if is_in_ci():
# Update last result with speed data
if TEST_RESULTS and TEST_RESULTS[-1]["variant"] == "tp_mtp":
TEST_RESULTS[-1]["speed"] = speed
# Write the summary table after all tests complete
_write_summary_table()
self.assertGreater(acc_length, 2.5)
self.assertGreater(speed, 110)
def _format_optional_metric(value, fmt=".2f", suffix=""):
"""Format an optional metric value, returning '-' if not available."""
if value is None:
return "-"
return f"{value:{fmt}}{suffix}"
def _write_summary_table():
"""Write a markdown table with all test results."""
if not TEST_RESULTS:
return
gpu_config = os.getenv("GPU_CONFIG", "8-gpu-h200")
# Build table header - keep original columns + add MTP-specific ones
summary = (
f"### {DEEPSEEK_V32_MODEL_PATH} GSM8K Accuracy (TP Tests) [{gpu_config}]\n\n"
)
summary += "| Variant | Prefill Backend | Decode Backend | KV Cache | Accuracy | Spec Acc Len | Speed |\n"
summary += "|---------|-----------------|----------------|----------|----------|--------------|-------|\n"
# Add each result as a row
for result in TEST_RESULTS:
summary += (
f"| {result['variant']} | {result['prefill_backend']} | "
f"{result['decode_backend']} | {result['kv_cache']} | "
f"{result['accuracy']:.3f} | "
f"{_format_optional_metric(result.get('avg_spec_accept_length'))} | "
f"{_format_optional_metric(result.get('speed'), '.1f', ' tok/s')} |\n"
)
write_github_step_summary(summary)
if __name__ == "__main__":
unittest.main()

49
test/nightly/test_glm_4_6_perf.py
View File

@@ -1,49 +0,0 @@
import unittest
from nightly_utils import NightlyBenchmarkRunner
from sglang.test.test_utils import DEFAULT_URL_FOR_TEST, _parse_int_list_env
GLM_4_6_MODEL_PATH = "zai-org/GLM-4.6"
PROFILE_DIR = "performance_profiles_glm_4_6"
class TestNightlyGLM46Performance(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.model = GLM_4_6_MODEL_PATH
cls.base_url = DEFAULT_URL_FOR_TEST
cls.batch_sizes = [1, 1, 8, 16, 64]
cls.input_lens = tuple(_parse_int_list_env("NIGHTLY_INPUT_LENS", "4096"))
cls.output_lens = tuple(_parse_int_list_env("NIGHTLY_OUTPUT_LENS", "512"))
# GLM-4.6 is a 357B MoE model
cls.other_args = [
"--tp",
"8",
"--trust-remote-code",
]
cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url)
cls.runner.setup_profile_directory()
def test_bench_one_batch(self):
results, success = self.runner.run_benchmark_for_model(
model_path=self.model,
batch_sizes=self.batch_sizes,
input_lens=self.input_lens,
output_lens=self.output_lens,
other_args=self.other_args,
)
self.runner.add_report(results)
self.runner.write_final_report()
if not success:
raise AssertionError(
f"Benchmark failed for {self.model}. Check the logs for details."
)
if __name__ == "__main__":
unittest.main()

54
test/nightly/test_kimi_k2_thinking_perf.py
View File

@@ -1,54 +0,0 @@
import unittest
from nightly_utils import NightlyBenchmarkRunner
from sglang.test.test_utils import DEFAULT_URL_FOR_TEST, _parse_int_list_env
KIMI_K2_THINKING_MODEL_PATH = "moonshotai/Kimi-K2-Thinking"
PROFILE_DIR = "performance_profiles_kimi_k2_thinking"
class TestNightlyKimiK2ThinkingPerformance(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.model = KIMI_K2_THINKING_MODEL_PATH
cls.base_url = DEFAULT_URL_FOR_TEST
cls.batch_sizes = [1, 1, 8, 16, 64]
cls.input_lens = tuple(_parse_int_list_env("NIGHTLY_INPUT_LENS", "4096"))
cls.output_lens = tuple(_parse_int_list_env("NIGHTLY_OUTPUT_LENS", "512"))
# Kimi-K2-Thinking requires specific launch arguments
cls.other_args = [
"--tp",
"8",
"--trust-remote-code",
"--tool-call-parser",
"kimi_k2",
"--reasoning-parser",
"kimi_k2",
]
cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url)
cls.runner.setup_profile_directory()
def test_bench_one_batch(self):
results, success = self.runner.run_benchmark_for_model(
model_path=self.model,
batch_sizes=self.batch_sizes,
input_lens=self.input_lens,
output_lens=self.output_lens,
other_args=self.other_args,
extra_bench_args=["--trust-remote-code"],
)
self.runner.add_report(results)
self.runner.write_final_report()
if not success:
raise AssertionError(
f"Benchmark failed for {self.model}. Check the logs for details."
)
if __name__ == "__main__":
unittest.main()

54
test/nightly/test_minimax_m2_perf.py
View File

@@ -1,54 +0,0 @@
import unittest
from nightly_utils import NightlyBenchmarkRunner
from sglang.test.test_utils import DEFAULT_URL_FOR_TEST, _parse_int_list_env
MINIMAX_M2_MODEL_PATH = "MiniMaxAI/MiniMax-M2"
PROFILE_DIR = "performance_profiles_minimax_m2"
class TestNightlyMiniMaxM2Performance(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.model = MINIMAX_M2_MODEL_PATH
cls.base_url = DEFAULT_URL_FOR_TEST
cls.batch_sizes = [1, 1, 8, 16, 64]
cls.input_lens = tuple(_parse_int_list_env("NIGHTLY_INPUT_LENS", "4096"))
cls.output_lens = tuple(_parse_int_list_env("NIGHTLY_OUTPUT_LENS", "512"))
# MiniMax-M2 is a 230B MoE model with 10B active params
cls.other_args = [
"--tp",
"8",
"--ep",
"8",
"--model-loader-extra-config",
'{"enable_multithread_load": true}',
"--trust-remote-code",
]
cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url)
cls.runner.setup_profile_directory()
def test_bench_one_batch(self):
results, success = self.runner.run_benchmark_for_model(
model_path=self.model,
batch_sizes=self.batch_sizes,
input_lens=self.input_lens,
output_lens=self.output_lens,
other_args=self.other_args,
extra_bench_args=["--trust-remote-code"],
)
self.runner.add_report(results)
self.runner.write_final_report()
if not success:
raise AssertionError(
f"Benchmark failed for {self.model}. Check the logs for details."
)
if __name__ == "__main__":
unittest.main()

201
test/nightly/test_mistral_large3_perf.py
View File

@@ -1,201 +0,0 @@
import os
import unittest
from types import SimpleNamespace
from nightly_utils import NightlyBenchmarkRunner
from sglang.srt.utils import kill_process_tree
from sglang.test.run_eval import run_eval
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
_parse_int_list_env,
popen_launch_server,
)
MISTRAL_LARGE3_MODEL_PATH = "mistralai/Mistral-Large-3-675B-Instruct-2512"
MISTRAL_LARGE3_EAGLE_MODEL_PATH = "mistralai/Mistral-Large-3-675B-Instruct-2512-Eagle"
PROFILE_DIR = "performance_profiles_mistral_large3"
class TestNightlyMistralLarge3Performance(unittest.TestCase):
@classmethod
def setUpClass(cls):
# Set environment variable to disable JIT DeepGemm
os.environ["SGLANG_ENABLE_JIT_DEEPGEMM"] = "0"
cls.model = MISTRAL_LARGE3_MODEL_PATH
cls.base_url = DEFAULT_URL_FOR_TEST
cls.batch_sizes = [1, 1, 8, 16, 64]
cls.input_lens = tuple(_parse_int_list_env("NIGHTLY_INPUT_LENS", "4096"))
cls.output_lens = tuple(_parse_int_list_env("NIGHTLY_OUTPUT_LENS", "512"))
# Mistral-Large-3-675B requires TP=8 and trtllm_mla attention backend
cls.other_args = [
"--tp",
"8",
"--attention-backend",
"trtllm_mla",
"--model-loader-extra-config",
'{"enable_multithread_load": true}',
"--chat-template",
"mistral",
]
cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url)
cls.runner.setup_profile_directory()
@classmethod
def tearDownClass(cls):
# Clean up environment variable
if "SGLANG_ENABLE_JIT_DEEPGEMM" in os.environ:
del os.environ["SGLANG_ENABLE_JIT_DEEPGEMM"]
def test_bench_one_batch(self):
results, success = self.runner.run_benchmark_for_model(
model_path=self.model,
batch_sizes=self.batch_sizes,
input_lens=self.input_lens,
output_lens=self.output_lens,
other_args=self.other_args,
)
self.runner.add_report(results)
self.runner.write_final_report()
if not success:
raise AssertionError(
f"Benchmark failed for {self.model}. Check the logs for details."
)
def test_accuracy_mgsm(self):
"""Run MGSM accuracy evaluation for Mistral Large 3."""
process = popen_launch_server(
model=self.model,
base_url=self.base_url,
other_args=self.other_args,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
)
try:
args = SimpleNamespace(
base_url=self.base_url,
model=self.model,
eval_name="mgsm_en",
num_examples=None,
num_threads=1024,
)
metrics = run_eval(args)
print(f"MGSM accuracy for {self.model}: {metrics['score']}")
# Placeholder threshold - adjust after first successful run
expected_threshold = 0.90
self.assertGreaterEqual(
metrics["score"],
expected_threshold,
f"MGSM accuracy {metrics['score']} below threshold {expected_threshold}",
)
finally:
kill_process_tree(process.pid)
class TestNightlyMistralLarge3EaglePerformance(unittest.TestCase):
"""Test Mistral Large 3 with Eagle speculative decoding."""
@classmethod
def setUpClass(cls):
# Set environment variable to disable JIT DeepGemm
os.environ["SGLANG_ENABLE_JIT_DEEPGEMM"] = "0"
cls.model = MISTRAL_LARGE3_MODEL_PATH
cls.base_url = DEFAULT_URL_FOR_TEST
cls.batch_sizes = [1, 1, 8, 16, 64]
cls.input_lens = tuple(_parse_int_list_env("NIGHTLY_INPUT_LENS", "4096"))
cls.output_lens = tuple(_parse_int_list_env("NIGHTLY_OUTPUT_LENS", "512"))
# Mistral-Large-3 with Eagle speculative decoding
# Eagle model is used as draft model for speculative decoding
cls.other_args = [
"--tp",
"8",
"--attention-backend",
"trtllm_mla",
"--speculative-algorithm",
"EAGLE",
"--speculative-draft-model-path",
MISTRAL_LARGE3_EAGLE_MODEL_PATH,
"--speculative-num-steps",
"3",
"--speculative-eagle-topk",
"1",
"--speculative-num-draft-tokens",
"4",
"--kv-cache-dtype",
"auto",
"--model-loader-extra-config",
'{"enable_multithread_load": true}',
"--chat-template",
"mistral",
]
cls.runner = NightlyBenchmarkRunner(
"performance_profiles_mistral_large3_eagle", cls.__name__, cls.base_url
)
cls.runner.setup_profile_directory()
@classmethod
def tearDownClass(cls):
# Clean up environment variable
if "SGLANG_ENABLE_JIT_DEEPGEMM" in os.environ:
del os.environ["SGLANG_ENABLE_JIT_DEEPGEMM"]
def test_eagle_bench_one_batch(self):
results, success = self.runner.run_benchmark_for_model(
model_path=self.model,
batch_sizes=self.batch_sizes,
input_lens=self.input_lens,
output_lens=self.output_lens,
other_args=self.other_args,
)
self.runner.add_report(results)
self.runner.write_final_report()
if not success:
raise AssertionError(
f"Benchmark failed for {self.model} with Eagle. Check the logs for details."
)
def test_eagle_accuracy_mgsm(self):
"""Run MGSM accuracy evaluation for Mistral Large 3 with Eagle."""
process = popen_launch_server(
model=self.model,
base_url=self.base_url,
other_args=self.other_args,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
)
try:
args = SimpleNamespace(
base_url=self.base_url,
model=self.model,
eval_name="mgsm_en",
num_examples=None,
num_threads=1024,
)
metrics = run_eval(args)
print(f"MGSM accuracy for {self.model} with Eagle: {metrics['score']}")
# Placeholder threshold - adjust after first successful run
expected_threshold = 0.90
self.assertGreaterEqual(
metrics["score"],
expected_threshold,
f"MGSM accuracy {metrics['score']} below threshold {expected_threshold}",
)
finally:
kill_process_tree(process.pid)
if __name__ == "__main__":
unittest.main()

49
test/nightly/test_qwen3_235b_perf.py
View File

@@ -1,49 +0,0 @@
import unittest
from nightly_utils import NightlyBenchmarkRunner
from sglang.test.test_utils import DEFAULT_URL_FOR_TEST, _parse_int_list_env
QWEN3_235B_MODEL_PATH = "Qwen/Qwen3-235B-A22B-Instruct-2507"
PROFILE_DIR = "performance_profiles_qwen3_235b"
class TestNightlyQwen3235BPerformance(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.model = QWEN3_235B_MODEL_PATH
cls.base_url = DEFAULT_URL_FOR_TEST
cls.batch_sizes = [1, 1, 8, 16, 64]
cls.input_lens = tuple(_parse_int_list_env("NIGHTLY_INPUT_LENS", "4096"))
cls.output_lens = tuple(_parse_int_list_env("NIGHTLY_OUTPUT_LENS", "512"))
# Qwen3-235B requires TP=8 for 8 GPUs
cls.other_args = [
"--tp",
"8",
"--trust-remote-code",
]
cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url)
cls.runner.setup_profile_directory()
def test_bench_one_batch(self):
results, success = self.runner.run_benchmark_for_model(
model_path=self.model,
batch_sizes=self.batch_sizes,
input_lens=self.input_lens,
output_lens=self.output_lens,
other_args=self.other_args,
)
self.runner.add_report(results)
self.runner.write_final_report()
if not success:
raise AssertionError(
f"Benchmark failed for {self.model}. Check the logs for details."
)
if __name__ == "__main__":
unittest.main()

79
test/registered/8-gpu-models/test_deepseek_v31.py Normal file
View File

@@ -0,0 +1,79 @@
import sys
import unittest
from pathlib import Path
# Add nightly directory to path for run_combined_tests import
sys.path.insert(0, str(Path(__file__).parent.parent.parent / "nightly"))
from accuracy_test_runner import AccuracyTestParams
from performance_test_runner import PerformanceTestParams
from run_combined_tests import run_combined_tests
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.test_utils import ModelLaunchSettings, is_blackwell_system
# Runs on both H200 and B200 via nightly-8-gpu-common suite
register_cuda_ci(est_time=12000, suite="nightly-8-gpu-common", nightly=True)
DEEPSEEK_V31_MODEL_PATH = "deepseek-ai/DeepSeek-V3.1"
@unittest.skipIf(not is_blackwell_system(), "Requires B200")
class TestDeepseekV31Unified(unittest.TestCase):
"""Unified test class for DeepSeek-V3.1 performance and accuracy.
Two variants:
- basic: Standard TP=8
- mtp: TP=8 + EAGLE speculative decoding
Each variant runs BOTH:
- Performance test (using NightlyBenchmarkRunner)
- Accuracy test (using run_eval with mgsm_en)
"""
def test_deepseek_v31_all_variants(self):
"""Run performance and accuracy for all DeepSeek-V3.1 variants."""
# Define base arguments shared by most variants
base_args = [
"--tp=8",
"--trust-remote-code",
"--model-loader-extra-config",
'{"enable_multithread_load": true}',
]
mtp_args = [
"--speculative-algorithm=EAGLE",
"--speculative-num-steps=3",
"--speculative-eagle-topk=1",
"--speculative-num-draft-tokens=4",
"--mem-frac=0.7",
]
variants = [
# Variant: "basic" - Standard TP=8
ModelLaunchSettings(
DEEPSEEK_V31_MODEL_PATH,
tp_size=8,
extra_args=base_args,
),
# Variant: "mtp" - TP=8 + EAGLE speculative decoding
ModelLaunchSettings(
DEEPSEEK_V31_MODEL_PATH,
tp_size=8,
extra_args=base_args + mtp_args,
),
]
run_combined_tests(
models=variants,
test_name="DeepSeek-V3.1 Unified",
accuracy_params=AccuracyTestParams(
dataset="gsm8k", baseline_accuracy=0.935
),
performance_params=PerformanceTestParams(
profile_dir="performance_profiles_deepseek_v31",
),
)
if __name__ == "__main__":
unittest.main()

194
test/registered/8-gpu-models/test_deepseek_v32.py Normal file
View File

@@ -0,0 +1,194 @@
import sys
import unittest
from pathlib import Path
# Add nightly directory to path for run_combined_tests import
sys.path.insert(0, str(Path(__file__).parent.parent.parent / "nightly"))
from accuracy_test_runner import AccuracyTestParams
from performance_test_runner import PerformanceTestParams
from run_combined_tests import run_combined_tests
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.test_utils import ModelLaunchSettings, is_blackwell_system
register_cuda_ci(est_time=8000, suite="nightly-8-gpu-common", nightly=True)
DEEPSEEK_V32_MODEL_PATH = "deepseek-ai/DeepSeek-V3.2"
BASE_ARGS = [
"--trust-remote-code",
"--model-loader-extra-config",
'{"enable_multithread_load": true}',
]
DP_ARGS = [
"--tp=8",
"--dp=8",
"--enable-dp-attention",
]
# Accuracy thresholds
GSM8K_BASELINE = 0.935
GPQA_BASELINE = 0.835
class TestDeepseekV32Unified(unittest.TestCase):
"""Unified test class for DeepSeek V3.2 performance and accuracy.
Tests multiple variants with both performance and accuracy tests:
- dp: Standard TP=8 + DP=8 with dp-attention
- dp+mtp: DP + EAGLE speculative decoding
- tp: Pure TP=8 only
- tp+mtp: Pure TP=8 + EAGLE speculative decoding
"""
@unittest.skipIf(is_blackwell_system(), "Requires H200 system")
def test_deepseek_v32_all_variants(self):
"""Run performance and accuracy for all DeepSeek V3.2 variants."""
TP_ARGS = [
"--tp=8",
]
MTP_ARGS = [
"--speculative-algorithm=EAGLE",
"--speculative-num-steps=3",
"--speculative-eagle-topk=1",
"--speculative-num-draft-tokens=4",
"--mem-frac=0.7",
]
variants = [
# Variant: "dp" - Standard TP=8 + DP=8 with dp-attention
ModelLaunchSettings(
DEEPSEEK_V32_MODEL_PATH,
tp_size=8,
extra_args=BASE_ARGS + DP_ARGS,
),
# Variant: "dp+mtp" - DP + EAGLE speculative decoding
ModelLaunchSettings(
DEEPSEEK_V32_MODEL_PATH,
tp_size=8,
extra_args=BASE_ARGS + DP_ARGS + MTP_ARGS,
),
# Variant: "tp" - Pure TP=8 only
ModelLaunchSettings(
DEEPSEEK_V32_MODEL_PATH,
tp_size=8,
extra_args=BASE_ARGS + TP_ARGS,
),
# Variant: "tp+mtp" - Pure TP=8 + EAGLE speculative decoding
ModelLaunchSettings(
DEEPSEEK_V32_MODEL_PATH,
tp_size=8,
extra_args=BASE_ARGS + TP_ARGS + MTP_ARGS,
),
]
run_combined_tests(
models=variants,
test_name="DeepSeek-V3.2 Unified",
accuracy_params=AccuracyTestParams(
dataset="gsm8k", baseline_accuracy=GSM8K_BASELINE
),
performance_params=PerformanceTestParams(
batch_sizes=[1, 8, 16, 64],
profile_dir="performance_profiles_deepseek_v32",
),
)
@unittest.skipIf(is_blackwell_system(), "Requires H200 system")
def test_deepseek_v32_nsa_backends(self):
"""Test NSA attention backend variants (H200 only).
Tests three NSA backend configurations:
- flashmla: flashmla_sparse prefill + flashmla_kv decode
- fa3: FA3 prefill + FA3 decode
- fp8kvcache: default backends with FP8 KV cache
"""
NSA_FLASHMLA_ARGS = [
"--attention-backend=nsa",
"--nsa-prefill-backend=flashmla_sparse",
"--nsa-decode-backend=flashmla_kv",
]
NSA_FA3_ARGS = [
"--attention-backend=nsa",
"--nsa-prefill-backend=fa3",
"--nsa-decode-backend=fa3",
]
NSA_FP8KV_ARGS = [
"--attention-backend=nsa",
"--kv-cache-dtype=fp8_e4m3",
]
nsa_variants = [
# flashmla backend
ModelLaunchSettings(
DEEPSEEK_V32_MODEL_PATH,
tp_size=8,
extra_args=BASE_ARGS + DP_ARGS + NSA_FLASHMLA_ARGS,
),
# fa3 backend
ModelLaunchSettings(
DEEPSEEK_V32_MODEL_PATH,
tp_size=8,
extra_args=BASE_ARGS + DP_ARGS + NSA_FA3_ARGS,
),
# fp8 kv cache
ModelLaunchSettings(
DEEPSEEK_V32_MODEL_PATH,
tp_size=8,
extra_args=BASE_ARGS + DP_ARGS + NSA_FP8KV_ARGS,
),
]
run_combined_tests(
models=nsa_variants,
test_name="DeepSeek-V3.2 NSA Backends",
accuracy_params=AccuracyTestParams(
dataset="gsm8k", baseline_accuracy=GSM8K_BASELINE
),
performance_params=PerformanceTestParams(
batch_sizes=[1, 8, 16, 64],
profile_dir="performance_profiles_deepseek_v32_nsa",
),
)
@unittest.skipIf(not is_blackwell_system(), "Requires B200")
def test_deepseek_v32_b200(self):
"""Test DeepSeek V3.2 with GPQA evaluation using thinking mode (B200 only).
This test runs GPQA evaluation with the reasoning parser enabled.
"""
B200_REASONING_ARGS = [
"--tool-call-parser=deepseekv32",
"--reasoning-parser=deepseek-v3",
]
variants = [
ModelLaunchSettings(
DEEPSEEK_V32_MODEL_PATH,
tp_size=8,
extra_args=BASE_ARGS + DP_ARGS + B200_REASONING_ARGS,
),
]
run_combined_tests(
models=variants,
test_name="DeepSeek-V3.2 GPQA (B200)",
accuracy_params=AccuracyTestParams(
dataset="gpqa",
baseline_accuracy=GPQA_BASELINE,
num_examples=198,
num_threads=198,
max_tokens=120000,
thinking_mode="deepseek-v3",
temperature=0.1,
repeat=4,
),
performance_params=None, # Skip performance test for GPQA
)
if __name__ == "__main__":
unittest.main()

57
test/registered/8-gpu-models/test_glm_46.py Normal file
View File

@@ -0,0 +1,57 @@
import sys
import unittest
from pathlib import Path
# Add nightly directory to path for run_combined_tests import
sys.path.insert(0, str(Path(__file__).parent.parent.parent / "nightly"))
from accuracy_test_runner import AccuracyTestParams
from performance_test_runner import PerformanceTestParams
from run_combined_tests import run_combined_tests
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.test_utils import ModelLaunchSettings
# Runs on both H200 and B200 via nightly-8-gpu-common suite
register_cuda_ci(est_time=12000, suite="nightly-8-gpu-common", nightly=True)
GLM_4_6_MODEL_PATH = "zai-org/GLM-4.6"
class TestGLM46Unified(unittest.TestCase):
"""Unified test class for GLM-4.6 performance and accuracy.
Single variant with simple TP=8 configuration.
GLM-4.6 is a 357B MoE model.
Runs BOTH:
- Performance test (using NightlyBenchmarkRunner)
- Accuracy test (using run_eval with mgsm_en)
"""
def test_glm_46(self):
"""Run performance and accuracy for GLM-4.6."""
base_args = [
"--tp=8",
"--trust-remote-code",
]
variants = [
ModelLaunchSettings(
GLM_4_6_MODEL_PATH,
tp_size=8,
extra_args=base_args,
),
]
run_combined_tests(
models=variants,
test_name="GLM-4.6 Unified",
accuracy_params=AccuracyTestParams(dataset="gsm8k", baseline_accuracy=0.80),
performance_params=PerformanceTestParams(
profile_dir="performance_profiles_glm_4_6",
),
)
if __name__ == "__main__":
unittest.main()

58
test/registered/8-gpu-models/test_kimi_k2.py Normal file
View File

@@ -0,0 +1,58 @@
import sys
import unittest
from pathlib import Path
# Add nightly directory to path for run_combined_tests import
sys.path.insert(0, str(Path(__file__).parent.parent.parent / "nightly"))
from accuracy_test_runner import AccuracyTestParams
from performance_test_runner import PerformanceTestParams
from run_combined_tests import run_combined_tests
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.test_utils import ModelLaunchSettings
# Runs on both H200 and B200 via nightly-8-gpu-common suite
register_cuda_ci(est_time=12000, suite="nightly-8-gpu-common", nightly=True)
KIMI_K2_THINKING_MODEL_PATH = "moonshotai/Kimi-K2-Thinking"
class TestKimiK2Unified(unittest.TestCase):
"""Unified test class for Kimi-K2-Thinking performance and accuracy.
Single variant with TP=8 + tool/reasoning parsers.
Runs BOTH:
- Performance test (using NightlyBenchmarkRunner with extra_bench_args)
- Accuracy test (using run_eval with mgsm_en)
"""
def test_kimi_k2(self):
"""Run performance and accuracy for Kimi-K2-Thinking."""
base_args = [
"--tp=8",
"--trust-remote-code",
"--tool-call-parser=kimi_k2",
"--reasoning-parser=kimi_k2",
]
variants = [
ModelLaunchSettings(
KIMI_K2_THINKING_MODEL_PATH,
tp_size=8,
extra_args=base_args,
),
]
run_combined_tests(
models=variants,
test_name="Kimi-K2-Thinking Unified",
accuracy_params=AccuracyTestParams(dataset="gsm8k", baseline_accuracy=0.95),
performance_params=PerformanceTestParams(
profile_dir="performance_profiles_kimi_k2_thinking",
),
)
if __name__ == "__main__":
unittest.main()

60
test/registered/8-gpu-models/test_minimax_m2.py Normal file
View File

@@ -0,0 +1,60 @@
import sys
import unittest
from pathlib import Path
# Add nightly directory to path for run_combined_tests import
sys.path.insert(0, str(Path(__file__).parent.parent.parent / "nightly"))
from accuracy_test_runner import AccuracyTestParams
from performance_test_runner import PerformanceTestParams
from run_combined_tests import run_combined_tests
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.test_utils import ModelLaunchSettings
# Runs on both H200 and B200 via nightly-8-gpu-common suite
register_cuda_ci(est_time=12000, suite="nightly-8-gpu-common", nightly=True)
MINIMAX_M2_MODEL_PATH = "MiniMaxAI/MiniMax-M2"
class TestMiniMaxM2Unified(unittest.TestCase):
"""Unified test class for MiniMax-M2 performance and accuracy.
Single variant with TP=8 + EP=8 configuration.
MiniMax-M2 is a 230B MoE model with 10B active params.
Runs BOTH:
- Performance test (using NightlyBenchmarkRunner with extra_bench_args)
- Accuracy test (using run_eval with mgsm_en)
"""
def test_minimax_m2(self):
"""Run performance and accuracy for MiniMax-M2."""
base_args = [
"--tp=8",
"--ep=8",
"--trust-remote-code",
"--model-loader-extra-config",
'{"enable_multithread_load": true}',
]
variants = [
ModelLaunchSettings(
MINIMAX_M2_MODEL_PATH,
tp_size=8,
extra_args=base_args,
),
]
run_combined_tests(
models=variants,
test_name="MiniMax-M2 Unified",
accuracy_params=AccuracyTestParams(dataset="gsm8k", baseline_accuracy=0.80),
performance_params=PerformanceTestParams(
profile_dir="performance_profiles_minimax_m2",
),
)
if __name__ == "__main__":
unittest.main()

92
test/registered/8-gpu-models/test_mistral_large3.py Normal file
View File

@@ -0,0 +1,92 @@
import os
import sys
import unittest
from pathlib import Path
# Add nightly directory to path for run_combined_tests import
sys.path.insert(0, str(Path(__file__).parent.parent.parent / "nightly"))
from accuracy_test_runner import AccuracyTestParams
from performance_test_runner import PerformanceTestParams
from run_combined_tests import run_combined_tests
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.test_utils import ModelLaunchSettings, is_blackwell_system
# Runs on both H200 and B200 via nightly-8-gpu-common suite
# Note: trtllm_mla backend may have hardware-specific behavior
register_cuda_ci(est_time=12000, suite="nightly-8-gpu-common", nightly=True)
MISTRAL_LARGE3_MODEL_PATH = "mistralai/Mistral-Large-3-675B-Instruct-2512"
MISTRAL_LARGE3_EAGLE_MODEL_PATH = "mistralai/Mistral-Large-3-675B-Instruct-2512-Eagle"
@unittest.skipIf(not is_blackwell_system(), "Requires B200")
class TestMistralLarge3Unified(unittest.TestCase):
"""Unified test class for Mistral-Large-3 performance and accuracy.
Two variants:
- basic: TP=8 + trtllm_mla backend
- eagle: basic + EAGLE speculative decoding with draft model
Each variant runs BOTH:
- Performance test (using NightlyBenchmarkRunner)
- Accuracy test (using run_eval with mgsm_en)
"""
@classmethod
def setUpClass(cls):
# Set environment variable to disable JIT DeepGemm
os.environ["SGLANG_ENABLE_JIT_DEEPGEMM"] = "0"
@classmethod
def tearDownClass(cls):
# Clean up environment variable
if "SGLANG_ENABLE_JIT_DEEPGEMM" in os.environ:
del os.environ["SGLANG_ENABLE_JIT_DEEPGEMM"]
def test_mistral_large3_all_variants(self):
"""Run performance and accuracy for all Mistral-Large-3 variants."""
base_args = [
"--tp=8",
"--attention-backend=trtllm_mla",
"--model-loader-extra-config",
'{"enable_multithread_load": true}',
"--chat-template=mistral",
]
eagle_args = [
"--speculative-algorithm=EAGLE",
f"--speculative-draft-model-path={MISTRAL_LARGE3_EAGLE_MODEL_PATH}",
"--speculative-num-steps=3",
"--speculative-eagle-topk=1",
"--speculative-num-draft-tokens=4",
"--kv-cache-dtype=auto",
]
variants = [
# Variant: "basic" - TP=8 + trtllm_mla backend
ModelLaunchSettings(
MISTRAL_LARGE3_MODEL_PATH,
tp_size=8,
extra_args=base_args,
),
# Variant: "eagle" - TP=8 + trtllm_mla + EAGLE with draft model
ModelLaunchSettings(
MISTRAL_LARGE3_MODEL_PATH,
tp_size=8,
extra_args=base_args + eagle_args,
),
]
run_combined_tests(
models=variants,
test_name="Mistral-Large-3 Unified",
accuracy_params=AccuracyTestParams(dataset="gsm8k", baseline_accuracy=0.90),
performance_params=PerformanceTestParams(
profile_dir="performance_profiles_mistral_large3",
),
)
if __name__ == "__main__":
unittest.main()

57
test/registered/8-gpu-models/test_qwen3_235b.py Normal file
View File

@@ -0,0 +1,57 @@
import sys
import unittest
from pathlib import Path
# Add nightly directory to path for run_combined_tests import
sys.path.insert(0, str(Path(__file__).parent.parent.parent / "nightly"))
from accuracy_test_runner import AccuracyTestParams
from performance_test_runner import PerformanceTestParams
from run_combined_tests import run_combined_tests
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.test_utils import ModelLaunchSettings, is_blackwell_system
# Runs on both H200 and B200 via nightly-8-gpu-common suite
register_cuda_ci(est_time=12000, suite="nightly-8-gpu-common", nightly=True)
QWEN3_235B_MODEL_PATH = "Qwen/Qwen3-235B-A22B-Instruct-2507"
@unittest.skipIf(not is_blackwell_system(), "Requires B200")
class TestQwen3235BUnified(unittest.TestCase):
"""Unified test class for Qwen3-235B performance and accuracy.
Single variant with simple TP=8 configuration.
Runs BOTH:
- Performance test (using NightlyBenchmarkRunner)
- Accuracy test (using run_eval with mgsm_en)
"""
def test_qwen3_235b(self):
"""Run performance and accuracy for Qwen3-235B."""
base_args = [
"--tp=8",
"--trust-remote-code",
]
variants = [
ModelLaunchSettings(
QWEN3_235B_MODEL_PATH,
tp_size=8,
extra_args=base_args,
),
]
run_combined_tests(
models=variants,
test_name="Qwen3-235B Unified",
accuracy_params=AccuracyTestParams(dataset="gsm8k", baseline_accuracy=0.88),
performance_params=PerformanceTestParams(
profile_dir="performance_profiles_qwen3_235b",
),
)
if __name__ == "__main__":
unittest.main()

3
test/run_suite.py
View File

@@ -39,6 +39,9 @@ NIGHTLY_SUITES = {
"nightly-8-gpu-h200",
"nightly-8-gpu-h20",
"nightly-8-gpu-b200",
"nightly-8-gpu-h200-basic", # Basic tests for large models on H200
"nightly-8-gpu-b200-basic", # Basic tests for large models on B200
"nightly-8-gpu-common", # Common tests that run on both H200 and B200
],
HWBackend.AMD: ["nightly-amd", "nightly-amd-8-gpu"],
HWBackend.CPU: [],