diff --git a/.github/workflows/nightly-test-nvidia.yml b/.github/workflows/nightly-test-nvidia.yml index 072e6cc69..dea2dfdab 100644 --- a/.github/workflows/nightly-test-nvidia.yml +++ b/.github/workflows/nightly-test-nvidia.yml @@ -108,6 +108,17 @@ jobs: run: | bash scripts/ci/ci_install_dependency.sh + - name: Run common 8-GPU model tests + if: always() + timeout-minutes: 200 + env: + TRACE_BASE_URL: https://raw.githubusercontent.com/sglang-bot/sglang-ci-data/main/traces/${{ github.run_id }} + PERFETTO_RELAY_URL: ${{ vars.PERFETTO_RELAY_URL }} + GPU_CONFIG: "8-gpu-h200" + run: | + cd test + python3 run_suite.py --hw cuda --suite nightly-8-gpu-common --nightly --timeout-per-file=12000 --continue-on-error + - name: Run test timeout-minutes: 30 env: @@ -116,82 +127,6 @@ jobs: cd test python3 run_suite.py --hw cuda --suite nightly-8-gpu-h200 --nightly --continue-on-error - - name: Run MiniMax-M2 nightly performance test - timeout-minutes: 180 - env: - TRACE_BASE_URL: https://raw.githubusercontent.com/sglang-bot/sglang-ci-data/main/traces/${{ github.run_id }} - PERFETTO_RELAY_URL: ${{ vars.PERFETTO_RELAY_URL }} - GPU_CONFIG: "8-gpu-h200" - run: | - rm -rf test/performance_profiles_minimax_m2/ - cd test - python3 nightly/test_minimax_m2_perf.py - - - name: Publish MiniMax-M2 traces to storage repo - env: - GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }} - GITHUB_RUN_ID: ${{ github.run_id }} - GITHUB_RUN_NUMBER: ${{ github.run_number }} - run: | - python3 scripts/ci/publish_traces.py --traces-dir test/performance_profiles_minimax_m2 - - - name: Run Qwen3-235B nightly performance test - timeout-minutes: 180 - env: - TRACE_BASE_URL: https://raw.githubusercontent.com/sglang-bot/sglang-ci-data/main/traces/${{ github.run_id }} - PERFETTO_RELAY_URL: ${{ vars.PERFETTO_RELAY_URL }} - GPU_CONFIG: "8-gpu-h200" - run: | - rm -rf test/performance_profiles_qwen3_235b/ - cd test - python3 nightly/test_qwen3_235b_perf.py - - - name: Publish Qwen3-235B traces to storage repo - env: - GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }} - GITHUB_RUN_ID: ${{ github.run_id }} - GITHUB_RUN_NUMBER: ${{ github.run_number }} - run: | - python3 scripts/ci/publish_traces.py --traces-dir test/performance_profiles_qwen3_235b - - - name: Run Kimi-K2-Thinking nightly performance test - timeout-minutes: 180 - env: - TRACE_BASE_URL: https://raw.githubusercontent.com/sglang-bot/sglang-ci-data/main/traces/${{ github.run_id }} - PERFETTO_RELAY_URL: ${{ vars.PERFETTO_RELAY_URL }} - GPU_CONFIG: "8-gpu-h200" - run: | - rm -rf test/performance_profiles_kimi_k2_thinking/ - cd test - python3 nightly/test_kimi_k2_thinking_perf.py - - - name: Publish Kimi-K2-Thinking traces to storage repo - env: - GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }} - GITHUB_RUN_ID: ${{ github.run_id }} - GITHUB_RUN_NUMBER: ${{ github.run_number }} - run: | - python3 scripts/ci/publish_traces.py --traces-dir test/performance_profiles_kimi_k2_thinking - - - name: Run GLM-4.6 nightly performance test - timeout-minutes: 180 - env: - TRACE_BASE_URL: https://raw.githubusercontent.com/sglang-bot/sglang-ci-data/main/traces/${{ github.run_id }} - PERFETTO_RELAY_URL: ${{ vars.PERFETTO_RELAY_URL }} - GPU_CONFIG: "8-gpu-h200" - run: | - rm -rf test/performance_profiles_glm_4_6/ - cd test - IS_BLACKWELL=1 python3 nightly/test_glm_4_6_perf.py - - - name: Publish GLM-4.6 traces to storage repo - env: - GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }} - GITHUB_RUN_ID: ${{ github.run_id }} - GITHUB_RUN_NUMBER: ${{ github.run_number }} - run: | - python3 scripts/ci/publish_traces.py --traces-dir test/performance_profiles_glm_4_6 - # General tests - 8 GPU H20 nightly-test-general-8-gpu-h20: if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-general-8-gpu-h20') @@ -446,147 +381,17 @@ jobs: run: | IS_BLACKWELL=1 bash scripts/ci/ci_install_dependency.sh - - name: Run Mistral-Large-3 nightly performance test - timeout-minutes: 180 + - name: Run common 8-GPU model tests + if: always() + timeout-minutes: 200 env: TRACE_BASE_URL: https://raw.githubusercontent.com/sglang-bot/sglang-ci-data/main/traces/${{ github.run_id }} PERFETTO_RELAY_URL: ${{ vars.PERFETTO_RELAY_URL }} GPU_CONFIG: "8-gpu-b200" SGLANG_ENABLE_JIT_DEEPGEMM: "0" run: | - rm -rf test/performance_profiles_mistral_large3/ - rm -rf test/performance_profiles_mistral_large3_eagle/ cd test - IS_BLACKWELL=1 python3 nightly/test_mistral_large3_perf.py - - - name: Publish Mistral-Large-3 traces to storage repo - env: - GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }} - GITHUB_RUN_ID: ${{ github.run_id }} - GITHUB_RUN_NUMBER: ${{ github.run_number }} - run: | - python3 scripts/ci/publish_traces.py --traces-dir test/performance_profiles_mistral_large3 - python3 scripts/ci/publish_traces.py --traces-dir test/performance_profiles_mistral_large3_eagle - - - name: Run DeepSeek v3.1 nightly performance test - if: always() - timeout-minutes: 180 - env: - TRACE_BASE_URL: https://raw.githubusercontent.com/sglang-bot/sglang-ci-data/main/traces/${{ github.run_id }} - PERFETTO_RELAY_URL: ${{ vars.PERFETTO_RELAY_URL }} - GPU_CONFIG: "8-gpu-b200" - run: | - rm -rf test/performance_profiles_deepseek_v31/ - cd test - IS_BLACKWELL=1 python3 nightly/test_deepseek_v31_perf.py - - - name: Publish DeepSeek v3.1 traces to storage repo - env: - GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }} - GITHUB_RUN_ID: ${{ github.run_id }} - GITHUB_RUN_NUMBER: ${{ github.run_number }} - run: | - python3 scripts/ci/publish_traces.py --traces-dir test/performance_profiles_deepseek_v31 - - - name: Run DeepSeek v3.2 nightly performance test - if: always() - timeout-minutes: 180 - env: - TRACE_BASE_URL: https://raw.githubusercontent.com/sglang-bot/sglang-ci-data/main/traces/${{ github.run_id }} - PERFETTO_RELAY_URL: ${{ vars.PERFETTO_RELAY_URL }} - GPU_CONFIG: "8-gpu-b200" - run: | - rm -rf test/performance_profiles_deepseek_v32/ - cd test - IS_BLACKWELL=1 python3 nightly/test_deepseek_v32_perf.py - - - name: Publish DeepSeek v3.2 traces to storage repo - env: - GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }} - GITHUB_RUN_ID: ${{ github.run_id }} - GITHUB_RUN_NUMBER: ${{ github.run_number }} - run: | - python3 scripts/ci/publish_traces.py --traces-dir test/performance_profiles_deepseek_v32 - - - name: Run Kimi-K2-Thinking nightly performance test - if: always() - timeout-minutes: 180 - env: - TRACE_BASE_URL: https://raw.githubusercontent.com/sglang-bot/sglang-ci-data/main/traces/${{ github.run_id }} - PERFETTO_RELAY_URL: ${{ vars.PERFETTO_RELAY_URL }} - GPU_CONFIG: "8-gpu-b200" - run: | - rm -rf test/performance_profiles_kimi_k2_thinking/ - cd test - IS_BLACKWELL=1 python3 nightly/test_kimi_k2_thinking_perf.py - - - name: Publish Kimi-K2-Thinking traces to storage repo - env: - GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }} - GITHUB_RUN_ID: ${{ github.run_id }} - GITHUB_RUN_NUMBER: ${{ github.run_number }} - run: | - python3 scripts/ci/publish_traces.py --traces-dir test/performance_profiles_kimi_k2_thinking - - - name: Run Qwen3-235B nightly performance test - if: always() - timeout-minutes: 180 - env: - TRACE_BASE_URL: https://raw.githubusercontent.com/sglang-bot/sglang-ci-data/main/traces/${{ github.run_id }} - PERFETTO_RELAY_URL: ${{ vars.PERFETTO_RELAY_URL }} - GPU_CONFIG: "8-gpu-b200" - run: | - rm -rf test/performance_profiles_qwen3_235b/ - cd test - IS_BLACKWELL=1 python3 nightly/test_qwen3_235b_perf.py - - - name: Publish Qwen3-235B traces to storage repo - env: - GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }} - GITHUB_RUN_ID: ${{ github.run_id }} - GITHUB_RUN_NUMBER: ${{ github.run_number }} - run: | - python3 scripts/ci/publish_traces.py --traces-dir test/performance_profiles_qwen3_235b - - - name: Run GLM-4.6 nightly performance test - if: always() - timeout-minutes: 180 - env: - TRACE_BASE_URL: https://raw.githubusercontent.com/sglang-bot/sglang-ci-data/main/traces/${{ github.run_id }} - PERFETTO_RELAY_URL: ${{ vars.PERFETTO_RELAY_URL }} - GPU_CONFIG: "8-gpu-b200" - run: | - rm -rf test/performance_profiles_glm_4_6/ - cd test - IS_BLACKWELL=1 python3 nightly/test_glm_4_6_perf.py - - - name: Publish GLM-4.6 traces to storage repo - env: - GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }} - GITHUB_RUN_ID: ${{ github.run_id }} - GITHUB_RUN_NUMBER: ${{ github.run_number }} - run: | - python3 scripts/ci/publish_traces.py --traces-dir test/performance_profiles_glm_4_6 - - - name: Run MiniMax-M2 nightly performance test - if: always() - timeout-minutes: 180 - env: - TRACE_BASE_URL: https://raw.githubusercontent.com/sglang-bot/sglang-ci-data/main/traces/${{ github.run_id }} - PERFETTO_RELAY_URL: ${{ vars.PERFETTO_RELAY_URL }} - GPU_CONFIG: "8-gpu-b200" - run: | - rm -rf test/performance_profiles_minimax_m2/ - cd test - IS_BLACKWELL=1 python3 nightly/test_minimax_m2_perf.py - - - name: Publish MiniMax-M2 traces to storage repo - env: - GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }} - GITHUB_RUN_ID: ${{ github.run_id }} - GITHUB_RUN_NUMBER: ${{ github.run_number }} - run: | - python3 scripts/ci/publish_traces.py --traces-dir test/performance_profiles_minimax_m2 + IS_BLACKWELL=1 python3 run_suite.py --hw cuda --suite nightly-8-gpu-common --nightly --timeout-per-file=12000 --continue-on-error # Final check job check-all-jobs: diff --git a/test/nightly/accuracy_test_runner.py b/test/nightly/accuracy_test_runner.py new file mode 100644 index 000000000..69a53a47c --- /dev/null +++ b/test/nightly/accuracy_test_runner.py @@ -0,0 +1,263 @@ +from dataclasses import dataclass +from types import SimpleNamespace +from typing import List, Optional, Tuple + +from sglang.srt.utils import kill_process_tree +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + ModelLaunchSettings, + popen_launch_server, + write_github_step_summary, +) + + +@dataclass +class AccuracyTestParams: + """Parameters for accuracy testing.""" + + dataset: str # e.g., "mgsm_en", "gsm8k", "mmmu", "gpqa" + baseline_accuracy: float # Required: minimum accuracy threshold + num_examples: Optional[int] = None + num_threads: Optional[int] = None + max_tokens: Optional[int] = None + return_latency: bool = False + # Extended parameters for special evaluations (e.g., GPQA with thinking mode) + thinking_mode: Optional[str] = None # e.g., "deepseek-v3" + temperature: Optional[float] = None + repeat: Optional[int] = None + + +@dataclass +class AccuracyTestResult: + """Result of an accuracy test.""" + + model: str + dataset: str + passed: bool + score: Optional[float] + baseline_accuracy: float + error: Optional[str] + latency: Optional[float] = None + + +def write_accuracy_github_summary( + test_name: str, + dataset: str, + results: List[AccuracyTestResult], +) -> None: + """Write accuracy test results to GitHub step summary. + + Args: + test_name: Name of the test + dataset: Dataset name used for evaluation + results: List of AccuracyTestResult objects + """ + summary = f"## {test_name} - Accuracy ({dataset})\n" + summary += "| model | status | score | baseline | error |\n" + summary += "| ----- | ------ | ----- | -------- | ----- |\n" + + for result in results: + status_emoji = "✅" if result.passed else "❌" + score_str = f"{result.score:.4f}" if result.score is not None else "N/A" + baseline_str = f"{result.baseline_accuracy:.4f}" + error_str = result.error if result.error else "-" + summary += f"| {result.model} | {status_emoji} | {score_str} | {baseline_str} | {error_str} |\n" + + write_github_step_summary(summary) + + +def _run_simple_eval( + model: ModelLaunchSettings, + base_url: str, + dataset: str, + num_examples: Optional[int] = None, + num_threads: Optional[int] = None, + max_tokens: Optional[int] = None, + return_latency: bool = False, + thinking_mode: Optional[str] = None, + temperature: Optional[float] = None, + repeat: Optional[int] = None, +) -> Tuple[bool, Optional[str], Optional[dict]]: + """Run evaluation using simple_eval backend (run_eval.py). + + Returns: + Tuple of (success, error_message, metrics_dict) + """ + process = None + try: + process = popen_launch_server( + model.model_path, + base_url, + other_args=model.extra_args, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + ) + + args = SimpleNamespace( + base_url=base_url, + model=model.model_path, + eval_name=dataset, + num_examples=num_examples, + num_threads=num_threads or 1024, + ) + + if max_tokens is not None: + args.max_tokens = max_tokens + + if return_latency: + args.return_latency = True + + if thinking_mode is not None: + args.thinking_mode = thinking_mode + + if temperature is not None: + args.temperature = temperature + + if repeat is not None: + args.repeat = repeat + + result = run_eval(args) + + # Handle result format (run_eval can return metrics or (metrics, latency)) + if return_latency and isinstance(result, tuple): + metrics, latency = result + metrics["latency"] = round(latency, 4) + else: + metrics = result + + return True, None, metrics + + except Exception as e: + return False, f"Accuracy test exception: {str(e)}", None + + finally: + if process: + kill_process_tree(process.pid) + + +def _run_few_shot_eval( + model: ModelLaunchSettings, + base_url: str, + num_questions: Optional[int] = None, + num_shots: int = 8, + max_tokens: int = 512, +) -> Tuple[bool, Optional[str], Optional[dict]]: + """Run evaluation using few_shot backend (few_shot_gsm8k.py). + + Returns: + Tuple of (success, error_message, metrics_dict) + """ + from sglang.test.few_shot_gsm8k import run_eval as run_few_shot_eval + + process = None + try: + process = popen_launch_server( + model.model_path, + base_url, + other_args=model.extra_args, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + ) + + args = SimpleNamespace( + num_shots=num_shots, + data_path=None, + num_questions=num_questions or 200, + max_new_tokens=max_tokens, + parallel=128, + host="http://127.0.0.1", + port=int(base_url.split(":")[-1]), + ) + + metrics = run_few_shot_eval(args) + + # Normalize metrics format (few_shot returns "accuracy", simple_eval returns "score") + if "accuracy" in metrics and "score" not in metrics: + metrics["score"] = metrics["accuracy"] + + return True, None, metrics + + except Exception as e: + return False, f"Few-shot evaluation exception: {str(e)}", None + + finally: + if process: + kill_process_tree(process.pid) + + +def run_accuracy_test( + model: ModelLaunchSettings, + params: AccuracyTestParams, + base_url: Optional[str] = None, +) -> AccuracyTestResult: + """Run accuracy test for a single model. + + Args: + model: ModelLaunchSettings with model config + params: AccuracyTestParams with dataset, baseline, and optional settings + base_url: Server base URL (default: DEFAULT_URL_FOR_TEST) + + Returns: + AccuracyTestResult with test outcome + """ + base_url = base_url or DEFAULT_URL_FOR_TEST + + print(f"\n{'='*60}") + print(f"Running ACCURACY test for {model.model_path}") + print(f" Dataset: {params.dataset}") + print(f" Baseline: {params.baseline_accuracy}") + print(f"{'='*60}\n") + + # Run evaluation based on dataset type + if params.dataset == "gsm8k": + success, error, metrics = _run_few_shot_eval( + model=model, + base_url=base_url, + num_questions=params.num_examples, + max_tokens=params.max_tokens or 512, + ) + else: + success, error, metrics = _run_simple_eval( + model=model, + base_url=base_url, + dataset=params.dataset, + num_examples=params.num_examples, + num_threads=params.num_threads, + max_tokens=params.max_tokens, + return_latency=params.return_latency, + thinking_mode=params.thinking_mode, + temperature=params.temperature, + repeat=params.repeat, + ) + + if not success: + print(f"✗ Accuracy test failed for {model.model_path}: {error}") + return AccuracyTestResult( + model=model.model_path, + dataset=params.dataset, + passed=False, + score=None, + baseline_accuracy=params.baseline_accuracy, + error=error, + ) + + # Validate against baseline + score = metrics.get("score", 0.0) + passed = score >= params.baseline_accuracy + latency = metrics.get("latency") + + if passed: + print(f"✓ Accuracy {score:.3f} >= baseline {params.baseline_accuracy:.3f}") + else: + error = f"Accuracy {score:.3f} below baseline {params.baseline_accuracy:.3f}" + print(f"✗ {error}") + + return AccuracyTestResult( + model=model.model_path, + dataset=params.dataset, + passed=passed, + score=score, + baseline_accuracy=params.baseline_accuracy, + error=error if not passed else None, + latency=latency, + ) diff --git a/test/nightly/nightly_utils.py b/test/nightly/nightly_utils.py index eb5af4ad4..48a5bfff8 100644 --- a/test/nightly/nightly_utils.py +++ b/test/nightly/nightly_utils.py @@ -6,6 +6,8 @@ import subprocess import time from typing import List, Optional, Tuple +import requests + from sglang.srt.utils import kill_process_tree from sglang.test.nightly_bench_utils import BenchmarkResult, generate_markdown_report from sglang.test.test_utils import ( @@ -211,7 +213,7 @@ class NightlyBenchmarkRunner: other_args: Optional[List[str]] = None, variant: str = "", extra_bench_args: Optional[List[str]] = None, - ) -> Tuple[List[BenchmarkResult], bool]: + ) -> Tuple[List[BenchmarkResult], bool, Optional[float]]: """Run a complete benchmark for a single model with server management. This method handles: @@ -219,6 +221,7 @@ class NightlyBenchmarkRunner: - Profile filename generation - Benchmark command construction and execution - Result loading and parsing + - Fetching speculative decoding accept length (for MTP/EAGLE) Args: model_path: Path to the model @@ -230,9 +233,10 @@ class NightlyBenchmarkRunner: extra_bench_args: Extra arguments for the benchmark command Returns: - Tuple of (list of BenchmarkResult objects, success_bool) + Tuple of (list of BenchmarkResult objects, success_bool, avg_spec_accept_length or None) """ benchmark_results = [] + avg_spec_accept_length = None model_description = f"{model_path}" + (f" ({variant})" if variant else "") # Launch server @@ -268,19 +272,42 @@ class NightlyBenchmarkRunner: result, cmd_success = self.run_benchmark_command(command, model_description) if not cmd_success: - return benchmark_results, False + return benchmark_results, False, None # Load results benchmark_results, load_success = self.load_benchmark_results( json_output_file, model_description ) - return benchmark_results, load_success + # Fetch speculative decoding accept length before killing server + avg_spec_accept_length = self._get_spec_accept_length() + + return benchmark_results, load_success, avg_spec_accept_length finally: # Always clean up server process kill_process_tree(process.pid) + def _get_spec_accept_length(self) -> Optional[float]: + """Query the server for avg_spec_accept_length metric. + + Returns: + The average speculative decoding accept length, or None if not available. + """ + try: + response = requests.get(f"{self.base_url}/get_server_info", timeout=10) + if response.status_code == 200: + server_info = response.json() + internal_states = server_info.get("internal_states", []) + if internal_states and len(internal_states) > 0: + accept_length = internal_states[0].get("avg_spec_accept_length") + if accept_length is not None: + print(f" avg_spec_accept_length={accept_length:.2f}") + return accept_length + except Exception as e: + print(f" Warning: Could not fetch spec accept length: {e}") + return None + def add_report(self, results: List[BenchmarkResult]) -> None: """Add benchmark results to the full report. diff --git a/test/nightly/performance_test_runner.py b/test/nightly/performance_test_runner.py new file mode 100644 index 000000000..48fb452cb --- /dev/null +++ b/test/nightly/performance_test_runner.py @@ -0,0 +1,231 @@ +from dataclasses import dataclass, field +from typing import List, Optional, Tuple + +from nightly_utils import NightlyBenchmarkRunner + +from sglang.test.nightly_bench_utils import BenchmarkResult +from sglang.test.test_utils import DEFAULT_URL_FOR_TEST, ModelLaunchSettings + + +@dataclass +class PerformanceTestParams: + """Parameters for performance testing.""" + + batch_sizes: List[int] = field(default_factory=lambda: [1, 8, 16, 64]) + input_lens: Tuple[int, ...] = (4096,) + output_lens: Tuple[int, ...] = (512,) + profile_dir: Optional[str] = None # None = auto-generate based on is_vlm + dataset_name: str = "mmmu" # For VLM perf test + # MTP/EAGLE speculative decoding: minimum accept length threshold (None = no validation) + spec_accept_length_threshold: Optional[float] = None + + +@dataclass +class PerformanceTestResult: + """Result of a performance test. + + Aggregates metrics across all batch sizes tested for a single model. + """ + + model: str + passed: bool + error: Optional[str] + # Aggregate metrics (from the largest batch size result, or None if failed) + latency: Optional[float] = None + input_throughput: Optional[float] = None + output_throughput: Optional[float] = None + overall_throughput: Optional[float] = None + # All individual benchmark results + benchmark_results: Optional[List[BenchmarkResult]] = None + # MTP/EAGLE speculative decoding metric + avg_spec_accept_length: Optional[float] = None + + +def run_performance_test( + model: ModelLaunchSettings, + perf_runner: NightlyBenchmarkRunner, + batch_sizes: List[int] = None, + input_lens: Tuple[int, ...] = (4096,), + output_lens: Tuple[int, ...] = (512,), + is_vlm: bool = False, + dataset_name: str = "mmmu", + spec_accept_length_threshold: Optional[float] = None, +) -> PerformanceTestResult: + + # Set default for mutable argument + if batch_sizes is None: + batch_sizes = [1, 8, 16, 64] + + print(f"\n{'='*60}") + print(f"Running PERFORMANCE test for {model.model_path}") + print(f" Batch sizes: {batch_sizes}") + print(f" Input lens: {input_lens}") + print(f" Output lens: {output_lens}") + if spec_accept_length_threshold is not None: + print(f" Spec accept length threshold: {spec_accept_length_threshold}") + print(f"{'='*60}\n") + + # Build extra args for benchmarks + extra_bench_args = ["--trust-remote-code"] + if is_vlm: + extra_bench_args.append(f"--dataset-name={dataset_name}") + + try: + results, success, avg_spec_accept_length = perf_runner.run_benchmark_for_model( + model_path=model.model_path, + batch_sizes=batch_sizes, + input_lens=input_lens, + output_lens=output_lens, + other_args=model.extra_args, + extra_bench_args=extra_bench_args, + ) + + if success and results: + perf_runner.add_report(results) + print(f"✓ Performance test succeeded for {model.model_path}") + + # Validate speculative decoding accept length if threshold is set + error_msg = None + passed = True + if spec_accept_length_threshold is not None: + if avg_spec_accept_length is None: + error_msg = f"Spec accept length threshold set but no accept length reported" + passed = False + print(f"✗ {error_msg}") + elif avg_spec_accept_length < spec_accept_length_threshold: + error_msg = ( + f"Spec accept length {avg_spec_accept_length:.2f} < " + f"threshold {spec_accept_length_threshold}" + ) + passed = False + print(f"✗ {error_msg}") + else: + print( + f"✓ Spec accept length {avg_spec_accept_length:.2f} >= " + f"threshold {spec_accept_length_threshold}" + ) + + # Extract aggregate metrics from the largest batch size result + largest_batch_result = max(results, key=lambda r: r.batch_size) + return PerformanceTestResult( + model=model.model_path, + passed=passed, + error=error_msg, + latency=largest_batch_result.latency, + input_throughput=largest_batch_result.input_throughput, + output_throughput=largest_batch_result.output_throughput, + overall_throughput=largest_batch_result.overall_throughput, + benchmark_results=results, + avg_spec_accept_length=avg_spec_accept_length, + ) + else: + error_msg = f"Performance test failed for {model.model_path}" + print(f"✗ {error_msg}") + return PerformanceTestResult( + model=model.model_path, + passed=False, + error=error_msg, + ) + + except Exception as e: + error_msg = f"Performance test exception for {model.model_path}: {str(e)}" + print(f"✗ {error_msg}") + return PerformanceTestResult( + model=model.model_path, + passed=False, + error=error_msg, + ) + + +def run_performance_for_models( + models: List[ModelLaunchSettings], + profile_dir: str, + test_name: str, + base_url: Optional[str] = None, + batch_sizes: List[int] = None, + input_lens: Tuple[int, ...] = (4096,), + output_lens: Tuple[int, ...] = (512,), + is_vlm: bool = False, + dataset_name: str = "mmmu", +) -> dict: + """Run performance tests for multiple models. + + Args: + models: List of ModelLaunchSettings to test + profile_dir: Directory for performance profiles + test_name: Name for the test (used in reports) + base_url: Server base URL (default: DEFAULT_URL_FOR_TEST) + batch_sizes: Batch sizes for perf test + input_lens: Input lengths + output_lens: Output lengths + is_vlm: Whether these are VLM models + dataset_name: Dataset name for VLM benchmarks + + Returns: + dict with results: + { + "all_passed": bool, + "results": [PerformanceTestResult, ...] + } + """ + base_url = base_url or DEFAULT_URL_FOR_TEST + + # Setup performance runner + perf_runner = NightlyBenchmarkRunner( + profile_dir=profile_dir, + test_name=test_name, + base_url=base_url, + ) + perf_runner.setup_profile_directory() + + all_results = [] + all_passed = True + + for model in models: + print("\n" + "=" * 80) + print(f"PERFORMANCE TEST: {model.model_path}") + print(f" TP Size: {model.tp_size}") + print(f" Extra Args: {model.extra_args}") + print("=" * 80) + + result = run_performance_test( + model=model, + perf_runner=perf_runner, + batch_sizes=batch_sizes, + input_lens=input_lens, + output_lens=output_lens, + is_vlm=is_vlm, + dataset_name=dataset_name, + ) + + all_results.append(result) + + if not result.passed: + all_passed = False + + # Write performance report + perf_runner.write_final_report() + + # Print summary + print("\n" + "=" * 60) + print(f"Performance Test Summary: {test_name}") + print("=" * 60) + for result in all_results: + status = "PASS" if result.passed else "FAIL" + throughput_str = ( + f", output: {result.output_throughput:.1f} tok/s" + if result.output_throughput + else "" + ) + print(f" {result.model}: {status}{throughput_str}") + if result.error: + print(f" Error: {result.error}") + + print("\n" + "=" * 60) + print(f"OVERALL: {'ALL PASSED' if all_passed else 'SOME FAILED'}") + print("=" * 60 + "\n") + + return { + "all_passed": all_passed, + "results": all_results, + } diff --git a/test/nightly/run_combined_tests.py b/test/nightly/run_combined_tests.py new file mode 100644 index 000000000..4e576753f --- /dev/null +++ b/test/nightly/run_combined_tests.py @@ -0,0 +1,192 @@ +from typing import List, Optional + +from accuracy_test_runner import ( + AccuracyTestParams, + AccuracyTestResult, + run_accuracy_test, + write_accuracy_github_summary, +) +from nightly_utils import NightlyBenchmarkRunner +from performance_test_runner import ( + PerformanceTestParams, + PerformanceTestResult, + run_performance_test, +) + +from sglang.test.test_utils import DEFAULT_URL_FOR_TEST, ModelLaunchSettings, is_in_ci + + +def run_combined_tests( + models: List[ModelLaunchSettings], + test_name: str = "NightlyTest", + base_url: Optional[str] = None, + is_vlm: bool = False, + accuracy_params: Optional[AccuracyTestParams] = None, + performance_params: Optional[PerformanceTestParams] = None, +) -> dict: + """Run performance and/or accuracy tests for a list of models. + + Args: + models: List of ModelLaunchSettings to test + test_name: Name for the test (used in reports) + base_url: Server base URL (default: DEFAULT_URL_FOR_TEST) + is_vlm: Whether these are VLM models (affects defaults) + accuracy_params: Parameters for accuracy tests (None to skip accuracy) + performance_params: Parameters for performance tests (None to skip perf) + + Returns: + dict with test results: + { + "all_passed": bool, + "results": [ + { + "model": str, + "perf_result": PerformanceTestResult/None, + "accuracy_result": AccuracyTestResult/None, + "errors": list, + }, + ... + ] + } + """ + base_url = base_url or DEFAULT_URL_FOR_TEST + run_perf = performance_params is not None + run_accuracy = accuracy_params is not None + + # Print test header + print("\n" + "=" * 80) + print(f"RUNNING: {test_name}") + print(f" Models: {len(models)}") + if run_accuracy: + print(f" Accuracy dataset: {accuracy_params.dataset}") + if run_perf: + print(f" Performance batches: {performance_params.batch_sizes}") + print("=" * 80) + + # Set up performance parameters + if run_perf: + perf = performance_params + profile_dir = perf.profile_dir or ( + "performance_profiles_vlms" + if is_vlm + else "performance_profiles_text_models" + ) + + perf_runner = NightlyBenchmarkRunner( + profile_dir=profile_dir, + test_name=test_name, + base_url=base_url, + ) + perf_runner.setup_profile_directory() + else: + perf_runner = None + + # Run tests for each model + all_results = [] + all_passed = True + + for model in models: + print("\n" + "=" * 80) + print(f"TESTING MODEL CONFIG: {model.model_path}") + print(f" TP Size: {model.tp_size}") + print(f" Extra Args: {model.extra_args}") + print("=" * 80) + + model_result = { + "model": model.model_path, + "perf_result": None, + "accuracy_result": None, + "errors": [], + } + + # Run performance test + if run_perf: + perf_result: PerformanceTestResult = run_performance_test( + model=model, + perf_runner=perf_runner, + batch_sizes=performance_params.batch_sizes, + input_lens=performance_params.input_lens, + output_lens=performance_params.output_lens, + is_vlm=is_vlm, + dataset_name=performance_params.dataset_name, + spec_accept_length_threshold=performance_params.spec_accept_length_threshold, + ) + model_result["perf_result"] = perf_result + if not perf_result.passed: + all_passed = False + model_result["errors"].append(perf_result.error) + + # Run accuracy test + if run_accuracy: + acc_result: AccuracyTestResult = run_accuracy_test( + model=model, + params=accuracy_params, + base_url=base_url, + ) + model_result["accuracy_result"] = acc_result + if not acc_result.passed: + all_passed = False + model_result["errors"].append(acc_result.error) + + all_results.append(model_result) + + # Write performance report if we ran perf tests + if run_perf and perf_runner: + perf_runner.write_final_report() + + # Write accuracy results to GitHub summary if in CI + if run_accuracy and is_in_ci(): + accuracy_results = [ + r["accuracy_result"] for r in all_results if r["accuracy_result"] + ] + write_accuracy_github_summary( + test_name, accuracy_params.dataset, accuracy_results + ) + + # Print summary + print("\n" + "=" * 60) + print(f"{test_name} Results Summary") + if run_accuracy: + print(f"Dataset: {accuracy_params.dataset}") + print(f"Baseline: {accuracy_params.baseline_accuracy}") + print("=" * 60) + for i, model_result in enumerate(all_results): + print(f"\nModel {i + 1}: {model_result['model']}") + if run_perf and model_result["perf_result"]: + perf = model_result["perf_result"] + throughput_str = ( + f", output: {perf.output_throughput:.1f} tok/s" + if perf.output_throughput + else "" + ) + accept_str = ( + f", accept_len: {perf.avg_spec_accept_length:.2f}" + if perf.avg_spec_accept_length + else "" + ) + print( + f" Performance: {'PASS' if perf.passed else 'FAIL'}{throughput_str}{accept_str}" + ) + if run_accuracy and model_result["accuracy_result"]: + acc = model_result["accuracy_result"] + print(f" Accuracy: {'PASS' if acc.passed else 'FAIL'}") + if acc.score is not None: + print(f" Score: {acc.score:.3f}") + if model_result["errors"]: + print(f" Errors: {model_result['errors']}") + + print("\n" + "=" * 60) + print(f"OVERALL: {'ALL TESTS PASSED' if all_passed else 'SOME TESTS FAILED'}") + print("=" * 60 + "\n") + + # Raise assertion error if any test failed + if not all_passed: + failed_models = [r["model"] for r in all_results if r["errors"]] + raise AssertionError( + f"Tests failed for models: {failed_models}. See results above for details." + ) + + return { + "all_passed": all_passed, + "results": all_results, + } diff --git a/test/nightly/test_deepseek_v31_perf.py b/test/nightly/test_deepseek_v31_perf.py deleted file mode 100644 index 58614350c..000000000 --- a/test/nightly/test_deepseek_v31_perf.py +++ /dev/null @@ -1,87 +0,0 @@ -import unittest - -from nightly_utils import NightlyBenchmarkRunner - -from sglang.test.test_utils import DEFAULT_URL_FOR_TEST, _parse_int_list_env - -DEEPSEEK_V31_MODEL_PATH = "deepseek-ai/DeepSeek-V3.1" -PROFILE_DIR = "performance_profiles_deepseek_v31" - - -class TestNightlyDeepseekV31Performance(unittest.TestCase): - @classmethod - def setUpClass(cls): - cls.model = DEEPSEEK_V31_MODEL_PATH - cls.base_url = DEFAULT_URL_FOR_TEST - cls.batch_sizes = [1, 1, 8, 16, 64] - cls.input_lens = tuple(_parse_int_list_env("NIGHTLY_INPUT_LENS", "4096")) - cls.output_lens = tuple(_parse_int_list_env("NIGHTLY_OUTPUT_LENS", "512")) - - # Define variant configurations - cls.variants = [ - { - "name": "basic", - "other_args": [ - "--trust-remote-code", - "--tp", - "8", - "--model-loader-extra-config", - '{"enable_multithread_load": true}', - ], - }, - { - "name": "mtp", - "other_args": [ - "--trust-remote-code", - "--tp", - "8", - "--speculative-algorithm", - "EAGLE", - "--speculative-num-steps", - "3", - "--speculative-eagle-topk", - "1", - "--speculative-num-draft-tokens", - "4", - "--mem-frac", - "0.7", - "--model-loader-extra-config", - '{"enable_multithread_load": true}', - ], - }, - ] - - cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url) - cls.runner.setup_profile_directory() - - def test_bench_one_batch(self): - failed_variants = [] - - try: - for variant_config in self.variants: - with self.subTest(variant=variant_config["name"]): - results, success = self.runner.run_benchmark_for_model( - model_path=self.model, - batch_sizes=self.batch_sizes, - input_lens=self.input_lens, - output_lens=self.output_lens, - other_args=variant_config["other_args"], - variant=variant_config["name"], - ) - - if not success: - failed_variants.append(variant_config["name"]) - - self.runner.add_report(results) - finally: - self.runner.write_final_report() - - if failed_variants: - raise AssertionError( - f"Benchmark failed for {self.model} with the following variants: " - f"{', '.join(failed_variants)}" - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/test/nightly/test_deepseek_v32_gpqa.py b/test/nightly/test_deepseek_v32_gpqa.py deleted file mode 100644 index f44e6efd4..000000000 --- a/test/nightly/test_deepseek_v32_gpqa.py +++ /dev/null @@ -1,102 +0,0 @@ -import unittest -from types import SimpleNamespace - -from sglang.srt.utils import kill_process_tree -from sglang.test.ci.ci_register import register_cuda_ci -from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k -from sglang.test.run_eval import run_eval -from sglang.test.test_utils import ( - DEFAULT_URL_FOR_TEST, - CustomTestCase, - is_in_ci, - popen_launch_server, - try_cached_model, - write_github_step_summary, -) - -register_cuda_ci(est_time=3600, suite="nightly-8-gpu-b200", nightly=True) - -# Use the latest version of DeepSeek-V3.2 -DEEPSEEK_V32_MODEL_PATH = "deepseek-ai/DeepSeek-V3.2" -SERVER_LAUNCH_TIMEOUT = 1200 - - -class TestDeepseekV32Accuracy(CustomTestCase): - @classmethod - def setUpClass(cls): - cls.model = try_cached_model(DEEPSEEK_V32_MODEL_PATH) - cls.base_url = DEFAULT_URL_FOR_TEST - other_args = [ - "--trust-remote-code", - "--tp", - "8", - "--enable-dp-attention", - "--dp", - "8", - "--tool-call-parser", - "deepseekv32", - "--reasoning-parser", - "deepseek-v3", - "--model-loader-extra-config", - '{"enable_multithread_load": true,"num_threads": 64}', - ] - cls.process = popen_launch_server( - cls.model, - cls.base_url, - timeout=SERVER_LAUNCH_TIMEOUT, - other_args=other_args, - ) - - @classmethod - def tearDownClass(cls): - kill_process_tree(cls.process.pid) - - def test_a_gsm8k( - self, - ): - args = SimpleNamespace( - num_shots=20, - data_path=None, - num_questions=1400, - parallel=1400, - max_new_tokens=512, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), - ) - metrics = run_eval_few_shot_gsm8k(args) - print(f"{metrics=}") - if is_in_ci(): - write_github_step_summary( - f"### test_gsm8k (deepseek-v32)\n" f'{metrics["accuracy"]=:.3f}\n' - ) - self.assertGreater(metrics["accuracy"], 0.935) - - def test_gpqa(self): - args = SimpleNamespace( - base_url=self.base_url, - model=DEEPSEEK_V32_MODEL_PATH, - eval_name="gpqa", - num_examples=198, - # use enough threads to allow parallelism - num_threads=198, - max_tokens=120000, - thinking_mode="deepseek-v3", - temperature=0.1, - # Repeat 4 times for shorter runtime. Ideally we should repeat at least 8 times. - repeat=4, - ) - - print(f"Evaluation start for gpqa") - metrics = run_eval(args) - print(f"Evaluation end for gpqa: {metrics=}, expected_score=0.835") - mean_score = metrics["mean_score"] - self.assertGreaterEqual(mean_score, 0.835) - - if is_in_ci(): - write_github_step_summary( - f"### test_gpqa (deepseek-v32)\n" f"Mean Score: {mean_score:.3f}\n" - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/test/nightly/test_deepseek_v32_nsabackend.py b/test/nightly/test_deepseek_v32_nsabackend.py deleted file mode 100644 index 45e2d665a..000000000 --- a/test/nightly/test_deepseek_v32_nsabackend.py +++ /dev/null @@ -1,224 +0,0 @@ -import os -import unittest -from types import SimpleNamespace - -from sglang.srt.utils import kill_process_tree -from sglang.test.ci.ci_register import register_cuda_ci -from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k -from sglang.test.test_utils import ( - DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - DEFAULT_URL_FOR_TEST, - CustomTestCase, - is_in_ci, - popen_launch_server, - write_github_step_summary, -) - -register_cuda_ci(est_time=600, suite="nightly-8-gpu-h200", nightly=True) - -DEEPSEEK_V32_MODEL_PATH = "deepseek-ai/DeepSeek-V3.2-Exp" - -# Global list to collect results -TEST_RESULTS = [] - - -class TestDeepseekV32NasBackend_flashmla(CustomTestCase): - @classmethod - def setUpClass(cls): - cls.model = DEEPSEEK_V32_MODEL_PATH - cls.base_url = DEFAULT_URL_FOR_TEST - other_args = [ - "--trust-remote-code", - "--attention-backend", - "nsa", - "--nsa-prefill-backend", - "flashmla_sparse", - "--nsa-decode-backend", - "flashmla_kv", - "--tp", - "8", - "--dp", - "8", - "--enable-dp-attention", - ] - cls.process = popen_launch_server( - cls.model, - cls.base_url, - timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - other_args=other_args, - ) - - @classmethod - def tearDownClass(cls): - kill_process_tree(cls.process.pid) - - def test_a_gsm8k( - self, - ): # Append an "a" to make this test run first (alphabetically) to warm up the server - args = SimpleNamespace( - num_shots=20, - data_path=None, - num_questions=1400, - parallel=1400, - max_new_tokens=512, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), - ) - metrics = run_eval_few_shot_gsm8k(args) - print(f"{metrics=}") - - if is_in_ci(): - TEST_RESULTS.append( - { - "variant": "flashmla", - "prefill_backend": "flashmla_sparse", - "decode_backend": "flashmla_kv", - "kv_cache": "fp16", - "accuracy": metrics["accuracy"], - } - ) - self.assertGreater(metrics["accuracy"], 0.935) - - -class TestDeepseekV32NasBackend_fa3(CustomTestCase): - @classmethod - def setUpClass(cls): - cls.model = DEEPSEEK_V32_MODEL_PATH - cls.base_url = DEFAULT_URL_FOR_TEST - other_args = [ - "--trust-remote-code", - "--attention-backend", - "nsa", - "--nsa-prefill-backend", - "fa3", - "--nsa-decode-backend", - "fa3", - "--tp", - "8", - "--dp", - "8", - "--enable-dp-attention", - ] - cls.process = popen_launch_server( - cls.model, - cls.base_url, - timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - other_args=other_args, - ) - - @classmethod - def tearDownClass(cls): - kill_process_tree(cls.process.pid) - - def test_a_gsm8k( - self, - ): # Append an "a" to make this test run first (alphabetically) to warm up the server - args = SimpleNamespace( - num_shots=20, - data_path=None, - num_questions=1400, - parallel=1400, - max_new_tokens=512, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), - ) - metrics = run_eval_few_shot_gsm8k(args) - print(f"{metrics=}") - - if is_in_ci(): - TEST_RESULTS.append( - { - "variant": "fa3", - "prefill_backend": "fa3", - "decode_backend": "fa3", - "kv_cache": "fp16", - "accuracy": metrics["accuracy"], - } - ) - self.assertGreater(metrics["accuracy"], 0.935) - - -class TestDeepseekV32NasBackend_fp8kvcache(CustomTestCase): - @classmethod - def setUpClass(cls): - cls.model = DEEPSEEK_V32_MODEL_PATH - cls.base_url = DEFAULT_URL_FOR_TEST - other_args = [ - "--trust-remote-code", - "--attention-backend", - "nsa", - "--kv-cache-dtype", - "fp8_e4m3", - "--tp", - "8", - "--dp", - "8", - "--enable-dp-attention", - ] - cls.process = popen_launch_server( - cls.model, - cls.base_url, - timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - other_args=other_args, - ) - - @classmethod - def tearDownClass(cls): - kill_process_tree(cls.process.pid) - - def test_a_gsm8k( - self, - ): # Append an "a" to make this test run first (alphabetically) to warm up the server - args = SimpleNamespace( - num_shots=20, - data_path=None, - num_questions=1400, - parallel=1400, - max_new_tokens=512, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), - ) - metrics = run_eval_few_shot_gsm8k(args) - print(f"{metrics=}") - - if is_in_ci(): - TEST_RESULTS.append( - { - "variant": "fp8kvcache", - "prefill_backend": "default", - "decode_backend": "default", - "kv_cache": "fp8_e4m3", - "accuracy": metrics["accuracy"], - } - ) - - # Write the summary table after all tests complete - _write_summary_table() - self.assertGreater(metrics["accuracy"], 0.935) - - -def _write_summary_table(): - """Write a markdown table with all test results.""" - if not TEST_RESULTS: - return - - gpu_config = os.getenv("GPU_CONFIG", "8-gpu-h200") - - # Build table header - summary = f"### {DEEPSEEK_V32_MODEL_PATH} GSM8K Accuracy [{gpu_config}]\n\n" - summary += "| Variant | Prefill Backend | Decode Backend | KV Cache | Accuracy |\n" - summary += "|---------|-----------------|----------------|----------|----------|\n" - - # Add each result as a row - for result in TEST_RESULTS: - summary += ( - f"| {result['variant']} | {result['prefill_backend']} | " - f"{result['decode_backend']} | {result['kv_cache']} | " - f"{result['accuracy']:.3f} |\n" - ) - - write_github_step_summary(summary) - - -if __name__ == "__main__": - unittest.main() diff --git a/test/nightly/test_deepseek_v32_perf.py b/test/nightly/test_deepseek_v32_perf.py deleted file mode 100644 index c90b835e4..000000000 --- a/test/nightly/test_deepseek_v32_perf.py +++ /dev/null @@ -1,123 +0,0 @@ -import unittest - -from nightly_utils import NightlyBenchmarkRunner - -from sglang.test.test_utils import DEFAULT_URL_FOR_TEST, _parse_int_list_env - -DEEPSEEK_V32_MODEL_PATH = "deepseek-ai/DeepSeek-V3.2-Exp" -PROFILE_DIR = "performance_profiles_deepseek_v32" - - -class TestNightlyDeepseekV32Performance(unittest.TestCase): - @classmethod - def setUpClass(cls): - cls.model = DEEPSEEK_V32_MODEL_PATH - cls.base_url = DEFAULT_URL_FOR_TEST - cls.batch_sizes = [1, 8, 16, 64] - cls.input_lens = tuple(_parse_int_list_env("NIGHTLY_INPUT_LENS", "4096")) - cls.output_lens = tuple(_parse_int_list_env("NIGHTLY_OUTPUT_LENS", "512")) - - # Define variant configurations - cls.variants = [ - { - "name": "dp", - "other_args": [ - "--trust-remote-code", - "--tp", - "8", - "--dp", - "8", - "--enable-dp-attention", - "--model-loader-extra-config", - '{"enable_multithread_load": true}', - ], - }, - { - "name": "dp+mtp", - "other_args": [ - "--trust-remote-code", - "--tp", - "8", - "--dp", - "8", - "--enable-dp-attention", - "--speculative-algorithm", - "EAGLE", - "--speculative-num-steps", - "3", - "--speculative-eagle-topk", - "1", - "--speculative-num-draft-tokens", - "4", - "--mem-frac", - "0.7", - "--model-loader-extra-config", - '{"enable_multithread_load": true}', - ], - }, - { - "name": "tp", - "other_args": [ - "--trust-remote-code", - "--tp", - "8", - "--model-loader-extra-config", - '{"enable_multithread_load": true}', - ], - }, - { - "name": "tp+mtp", - "other_args": [ - "--trust-remote-code", - "--tp", - "8", - "--speculative-algorithm", - "EAGLE", - "--speculative-num-steps", - "3", - "--speculative-eagle-topk", - "1", - "--speculative-num-draft-tokens", - "4", - "--mem-frac", - "0.7", - "--model-loader-extra-config", - '{"enable_multithread_load": true}', - ], - }, - ] - - cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url) - cls.runner.setup_profile_directory() - - def test_bench_one_batch(self): - failed_variants = [] - - try: - for variant_config in self.variants: - with self.subTest(variant=variant_config["name"]): - results, success = self.runner.run_benchmark_for_model( - model_path=self.model, - batch_sizes=self.batch_sizes, - input_lens=self.input_lens, - output_lens=self.output_lens, - other_args=variant_config["other_args"], - variant=variant_config["name"], - ) - - if not success: - failed_variants.append(variant_config["name"]) - - self.runner.add_report(results) - finally: - self.runner.write_final_report() - - if failed_variants: - raise AssertionError( - f"Benchmark failed for {self.model} with the following variants: " - f"{', '.join(failed_variants)}" - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/test/nightly/test_deepseek_v32_tp.py b/test/nightly/test_deepseek_v32_tp.py deleted file mode 100644 index 87c74428d..000000000 --- a/test/nightly/test_deepseek_v32_tp.py +++ /dev/null @@ -1,263 +0,0 @@ -import os -import unittest -from types import SimpleNamespace - -import requests - -from sglang.srt.utils import kill_process_tree -from sglang.test.ci.ci_register import register_cuda_ci -from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k -from sglang.test.send_one import BenchArgs, send_one_prompt -from sglang.test.test_utils import ( - DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - DEFAULT_URL_FOR_TEST, - CustomTestCase, - is_in_ci, - popen_launch_server, - write_github_step_summary, -) - -register_cuda_ci(est_time=900, suite="nightly-8-gpu-h200", nightly=True) - -DEEPSEEK_V32_MODEL_PATH = "deepseek-ai/DeepSeek-V3.2-Exp" - -# Global list to collect results -TEST_RESULTS = [] - - -class TestDeepseekV32_TP(CustomTestCase): - @classmethod - def setUpClass(cls): - cls.model = DEEPSEEK_V32_MODEL_PATH - cls.base_url = DEFAULT_URL_FOR_TEST - # Pure TP configuration without --dp and --enable-dp-attention - other_args = [ - "--trust-remote-code", - "--attention-backend", - "nsa", - "--nsa-prefill-backend", - "flashmla_sparse", - "--nsa-decode-backend", - "flashmla_kv", - "--tp", - "8", - ] - cls.process = popen_launch_server( - cls.model, - cls.base_url, - timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - other_args=other_args, - ) - - @classmethod - def tearDownClass(cls): - kill_process_tree(cls.process.pid) - - def test_a_gsm8k( - self, - ): # Append an "a" to make this test run first (alphabetically) to warm up the server - args = SimpleNamespace( - num_shots=20, - data_path=None, - num_questions=1400, - parallel=1400, - max_new_tokens=512, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), - ) - metrics = run_eval_few_shot_gsm8k(args) - print(f"{metrics=}") - - if is_in_ci(): - TEST_RESULTS.append( - { - "variant": "pure_tp", - "prefill_backend": "flashmla_sparse", - "decode_backend": "flashmla_kv", - "kv_cache": "fp16", - "accuracy": metrics["accuracy"], - } - ) - self.assertGreater(metrics["accuracy"], 0.935) - - -class TestDeepseekV32_Partial_TP(CustomTestCase): - @classmethod - def setUpClass(cls): - cls.model = DEEPSEEK_V32_MODEL_PATH - cls.base_url = DEFAULT_URL_FOR_TEST - # Partial TP configuration with dp=4 and dp-attention enabled - other_args = [ - "--trust-remote-code", - "--attention-backend", - "nsa", - "--nsa-prefill-backend", - "flashmla_sparse", - "--nsa-decode-backend", - "flashmla_kv", - "--tp", - "8", - "--dp", - "4", - "--enable-dp-attention", - ] - cls.process = popen_launch_server( - cls.model, - cls.base_url, - timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - other_args=other_args, - ) - - @classmethod - def tearDownClass(cls): - kill_process_tree(cls.process.pid) - - def test_a_gsm8k( - self, - ): # Append an "a" to make this test run first (alphabetically) to warm up the server - args = SimpleNamespace( - num_shots=20, - data_path=None, - num_questions=1400, - parallel=1400, - max_new_tokens=512, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), - ) - metrics = run_eval_few_shot_gsm8k(args) - print(f"{metrics=}") - - if is_in_ci(): - TEST_RESULTS.append( - { - "variant": "partial_tp", - "prefill_backend": "flashmla_sparse", - "decode_backend": "flashmla_kv", - "kv_cache": "fp16", - "accuracy": metrics["accuracy"], - } - ) - self.assertGreater(metrics["accuracy"], 0.935) - - -class TestDeepseekV32_TP_MTP(CustomTestCase): - """Test DeepSeek V3.2 with pure TP + MTP (EAGLE speculative decoding).""" - - @classmethod - def setUpClass(cls): - cls.model = DEEPSEEK_V32_MODEL_PATH - cls.base_url = DEFAULT_URL_FOR_TEST - other_args = [ - "--trust-remote-code", - "--tp", - "8", - "--speculative-algorithm", - "EAGLE", - "--speculative-num-steps", - "3", - "--speculative-eagle-topk", - "1", - "--speculative-num-draft-tokens", - "4", - "--mem-frac", - "0.7", - ] - cls.process = popen_launch_server( - cls.model, - cls.base_url, - timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - other_args=other_args, - ) - - @classmethod - def tearDownClass(cls): - kill_process_tree(cls.process.pid) - - def test_a_gsm8k(self): - requests.get(self.base_url + "/flush_cache") - args = SimpleNamespace( - num_shots=20, - data_path=None, - num_questions=1400, - parallel=1400, - max_new_tokens=512, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), - ) - metrics = run_eval_few_shot_gsm8k(args) - print(f"{metrics=}") - - server_info = requests.get(self.base_url + "/get_server_info") - avg_spec_accept_length = server_info.json()["internal_states"][0][ - "avg_spec_accept_length" - ] - print(f"{avg_spec_accept_length=}") - - if is_in_ci(): - TEST_RESULTS.append( - { - "variant": "tp_mtp", - "prefill_backend": "flashmla_sparse", - "decode_backend": "flashmla_kv", - "kv_cache": "fp16", - "accuracy": metrics["accuracy"], - "avg_spec_accept_length": avg_spec_accept_length, - } - ) - self.assertGreater(metrics["accuracy"], 0.935) - self.assertGreater(avg_spec_accept_length, 2.5) - - def test_bs_1_speed(self): - args = BenchArgs(port=int(self.base_url.split(":")[-1]), max_new_tokens=2048) - acc_length, speed = send_one_prompt(args) - - print(f"{acc_length=:.2f} {speed=:.2f}") - - if is_in_ci(): - # Update last result with speed data - if TEST_RESULTS and TEST_RESULTS[-1]["variant"] == "tp_mtp": - TEST_RESULTS[-1]["speed"] = speed - - # Write the summary table after all tests complete - _write_summary_table() - - self.assertGreater(acc_length, 2.5) - self.assertGreater(speed, 110) - - -def _format_optional_metric(value, fmt=".2f", suffix=""): - """Format an optional metric value, returning '-' if not available.""" - if value is None: - return "-" - return f"{value:{fmt}}{suffix}" - - -def _write_summary_table(): - """Write a markdown table with all test results.""" - if not TEST_RESULTS: - return - - gpu_config = os.getenv("GPU_CONFIG", "8-gpu-h200") - - # Build table header - keep original columns + add MTP-specific ones - summary = ( - f"### {DEEPSEEK_V32_MODEL_PATH} GSM8K Accuracy (TP Tests) [{gpu_config}]\n\n" - ) - summary += "| Variant | Prefill Backend | Decode Backend | KV Cache | Accuracy | Spec Acc Len | Speed |\n" - summary += "|---------|-----------------|----------------|----------|----------|--------------|-------|\n" - - # Add each result as a row - for result in TEST_RESULTS: - summary += ( - f"| {result['variant']} | {result['prefill_backend']} | " - f"{result['decode_backend']} | {result['kv_cache']} | " - f"{result['accuracy']:.3f} | " - f"{_format_optional_metric(result.get('avg_spec_accept_length'))} | " - f"{_format_optional_metric(result.get('speed'), '.1f', ' tok/s')} |\n" - ) - - write_github_step_summary(summary) - - -if __name__ == "__main__": - unittest.main() diff --git a/test/nightly/test_glm_4_6_perf.py b/test/nightly/test_glm_4_6_perf.py deleted file mode 100644 index a08df730a..000000000 --- a/test/nightly/test_glm_4_6_perf.py +++ /dev/null @@ -1,49 +0,0 @@ -import unittest - -from nightly_utils import NightlyBenchmarkRunner - -from sglang.test.test_utils import DEFAULT_URL_FOR_TEST, _parse_int_list_env - -GLM_4_6_MODEL_PATH = "zai-org/GLM-4.6" -PROFILE_DIR = "performance_profiles_glm_4_6" - - -class TestNightlyGLM46Performance(unittest.TestCase): - @classmethod - def setUpClass(cls): - cls.model = GLM_4_6_MODEL_PATH - cls.base_url = DEFAULT_URL_FOR_TEST - cls.batch_sizes = [1, 1, 8, 16, 64] - cls.input_lens = tuple(_parse_int_list_env("NIGHTLY_INPUT_LENS", "4096")) - cls.output_lens = tuple(_parse_int_list_env("NIGHTLY_OUTPUT_LENS", "512")) - - # GLM-4.6 is a 357B MoE model - cls.other_args = [ - "--tp", - "8", - "--trust-remote-code", - ] - - cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url) - cls.runner.setup_profile_directory() - - def test_bench_one_batch(self): - results, success = self.runner.run_benchmark_for_model( - model_path=self.model, - batch_sizes=self.batch_sizes, - input_lens=self.input_lens, - output_lens=self.output_lens, - other_args=self.other_args, - ) - - self.runner.add_report(results) - self.runner.write_final_report() - - if not success: - raise AssertionError( - f"Benchmark failed for {self.model}. Check the logs for details." - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/test/nightly/test_kimi_k2_thinking_perf.py b/test/nightly/test_kimi_k2_thinking_perf.py deleted file mode 100644 index aee862095..000000000 --- a/test/nightly/test_kimi_k2_thinking_perf.py +++ /dev/null @@ -1,54 +0,0 @@ -import unittest - -from nightly_utils import NightlyBenchmarkRunner - -from sglang.test.test_utils import DEFAULT_URL_FOR_TEST, _parse_int_list_env - -KIMI_K2_THINKING_MODEL_PATH = "moonshotai/Kimi-K2-Thinking" -PROFILE_DIR = "performance_profiles_kimi_k2_thinking" - - -class TestNightlyKimiK2ThinkingPerformance(unittest.TestCase): - @classmethod - def setUpClass(cls): - cls.model = KIMI_K2_THINKING_MODEL_PATH - cls.base_url = DEFAULT_URL_FOR_TEST - cls.batch_sizes = [1, 1, 8, 16, 64] - cls.input_lens = tuple(_parse_int_list_env("NIGHTLY_INPUT_LENS", "4096")) - cls.output_lens = tuple(_parse_int_list_env("NIGHTLY_OUTPUT_LENS", "512")) - - # Kimi-K2-Thinking requires specific launch arguments - cls.other_args = [ - "--tp", - "8", - "--trust-remote-code", - "--tool-call-parser", - "kimi_k2", - "--reasoning-parser", - "kimi_k2", - ] - - cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url) - cls.runner.setup_profile_directory() - - def test_bench_one_batch(self): - results, success = self.runner.run_benchmark_for_model( - model_path=self.model, - batch_sizes=self.batch_sizes, - input_lens=self.input_lens, - output_lens=self.output_lens, - other_args=self.other_args, - extra_bench_args=["--trust-remote-code"], - ) - - self.runner.add_report(results) - self.runner.write_final_report() - - if not success: - raise AssertionError( - f"Benchmark failed for {self.model}. Check the logs for details." - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/test/nightly/test_minimax_m2_perf.py b/test/nightly/test_minimax_m2_perf.py deleted file mode 100644 index 006069f2a..000000000 --- a/test/nightly/test_minimax_m2_perf.py +++ /dev/null @@ -1,54 +0,0 @@ -import unittest - -from nightly_utils import NightlyBenchmarkRunner - -from sglang.test.test_utils import DEFAULT_URL_FOR_TEST, _parse_int_list_env - -MINIMAX_M2_MODEL_PATH = "MiniMaxAI/MiniMax-M2" -PROFILE_DIR = "performance_profiles_minimax_m2" - - -class TestNightlyMiniMaxM2Performance(unittest.TestCase): - @classmethod - def setUpClass(cls): - cls.model = MINIMAX_M2_MODEL_PATH - cls.base_url = DEFAULT_URL_FOR_TEST - cls.batch_sizes = [1, 1, 8, 16, 64] - cls.input_lens = tuple(_parse_int_list_env("NIGHTLY_INPUT_LENS", "4096")) - cls.output_lens = tuple(_parse_int_list_env("NIGHTLY_OUTPUT_LENS", "512")) - - # MiniMax-M2 is a 230B MoE model with 10B active params - cls.other_args = [ - "--tp", - "8", - "--ep", - "8", - "--model-loader-extra-config", - '{"enable_multithread_load": true}', - "--trust-remote-code", - ] - - cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url) - cls.runner.setup_profile_directory() - - def test_bench_one_batch(self): - results, success = self.runner.run_benchmark_for_model( - model_path=self.model, - batch_sizes=self.batch_sizes, - input_lens=self.input_lens, - output_lens=self.output_lens, - other_args=self.other_args, - extra_bench_args=["--trust-remote-code"], - ) - - self.runner.add_report(results) - self.runner.write_final_report() - - if not success: - raise AssertionError( - f"Benchmark failed for {self.model}. Check the logs for details." - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/test/nightly/test_mistral_large3_perf.py b/test/nightly/test_mistral_large3_perf.py deleted file mode 100644 index 56aabbd6d..000000000 --- a/test/nightly/test_mistral_large3_perf.py +++ /dev/null @@ -1,201 +0,0 @@ -import os -import unittest -from types import SimpleNamespace - -from nightly_utils import NightlyBenchmarkRunner - -from sglang.srt.utils import kill_process_tree -from sglang.test.run_eval import run_eval -from sglang.test.test_utils import ( - DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - DEFAULT_URL_FOR_TEST, - _parse_int_list_env, - popen_launch_server, -) - -MISTRAL_LARGE3_MODEL_PATH = "mistralai/Mistral-Large-3-675B-Instruct-2512" -MISTRAL_LARGE3_EAGLE_MODEL_PATH = "mistralai/Mistral-Large-3-675B-Instruct-2512-Eagle" -PROFILE_DIR = "performance_profiles_mistral_large3" - - -class TestNightlyMistralLarge3Performance(unittest.TestCase): - @classmethod - def setUpClass(cls): - # Set environment variable to disable JIT DeepGemm - os.environ["SGLANG_ENABLE_JIT_DEEPGEMM"] = "0" - - cls.model = MISTRAL_LARGE3_MODEL_PATH - cls.base_url = DEFAULT_URL_FOR_TEST - cls.batch_sizes = [1, 1, 8, 16, 64] - cls.input_lens = tuple(_parse_int_list_env("NIGHTLY_INPUT_LENS", "4096")) - cls.output_lens = tuple(_parse_int_list_env("NIGHTLY_OUTPUT_LENS", "512")) - - # Mistral-Large-3-675B requires TP=8 and trtllm_mla attention backend - cls.other_args = [ - "--tp", - "8", - "--attention-backend", - "trtllm_mla", - "--model-loader-extra-config", - '{"enable_multithread_load": true}', - "--chat-template", - "mistral", - ] - - cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url) - cls.runner.setup_profile_directory() - - @classmethod - def tearDownClass(cls): - # Clean up environment variable - if "SGLANG_ENABLE_JIT_DEEPGEMM" in os.environ: - del os.environ["SGLANG_ENABLE_JIT_DEEPGEMM"] - - def test_bench_one_batch(self): - results, success = self.runner.run_benchmark_for_model( - model_path=self.model, - batch_sizes=self.batch_sizes, - input_lens=self.input_lens, - output_lens=self.output_lens, - other_args=self.other_args, - ) - - self.runner.add_report(results) - self.runner.write_final_report() - - if not success: - raise AssertionError( - f"Benchmark failed for {self.model}. Check the logs for details." - ) - - def test_accuracy_mgsm(self): - """Run MGSM accuracy evaluation for Mistral Large 3.""" - process = popen_launch_server( - model=self.model, - base_url=self.base_url, - other_args=self.other_args, - timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - ) - - try: - args = SimpleNamespace( - base_url=self.base_url, - model=self.model, - eval_name="mgsm_en", - num_examples=None, - num_threads=1024, - ) - metrics = run_eval(args) - print(f"MGSM accuracy for {self.model}: {metrics['score']}") - - # Placeholder threshold - adjust after first successful run - expected_threshold = 0.90 - self.assertGreaterEqual( - metrics["score"], - expected_threshold, - f"MGSM accuracy {metrics['score']} below threshold {expected_threshold}", - ) - finally: - kill_process_tree(process.pid) - - -class TestNightlyMistralLarge3EaglePerformance(unittest.TestCase): - """Test Mistral Large 3 with Eagle speculative decoding.""" - - @classmethod - def setUpClass(cls): - # Set environment variable to disable JIT DeepGemm - os.environ["SGLANG_ENABLE_JIT_DEEPGEMM"] = "0" - - cls.model = MISTRAL_LARGE3_MODEL_PATH - cls.base_url = DEFAULT_URL_FOR_TEST - cls.batch_sizes = [1, 1, 8, 16, 64] - cls.input_lens = tuple(_parse_int_list_env("NIGHTLY_INPUT_LENS", "4096")) - cls.output_lens = tuple(_parse_int_list_env("NIGHTLY_OUTPUT_LENS", "512")) - - # Mistral-Large-3 with Eagle speculative decoding - # Eagle model is used as draft model for speculative decoding - cls.other_args = [ - "--tp", - "8", - "--attention-backend", - "trtllm_mla", - "--speculative-algorithm", - "EAGLE", - "--speculative-draft-model-path", - MISTRAL_LARGE3_EAGLE_MODEL_PATH, - "--speculative-num-steps", - "3", - "--speculative-eagle-topk", - "1", - "--speculative-num-draft-tokens", - "4", - "--kv-cache-dtype", - "auto", - "--model-loader-extra-config", - '{"enable_multithread_load": true}', - "--chat-template", - "mistral", - ] - - cls.runner = NightlyBenchmarkRunner( - "performance_profiles_mistral_large3_eagle", cls.__name__, cls.base_url - ) - cls.runner.setup_profile_directory() - - @classmethod - def tearDownClass(cls): - # Clean up environment variable - if "SGLANG_ENABLE_JIT_DEEPGEMM" in os.environ: - del os.environ["SGLANG_ENABLE_JIT_DEEPGEMM"] - - def test_eagle_bench_one_batch(self): - results, success = self.runner.run_benchmark_for_model( - model_path=self.model, - batch_sizes=self.batch_sizes, - input_lens=self.input_lens, - output_lens=self.output_lens, - other_args=self.other_args, - ) - - self.runner.add_report(results) - self.runner.write_final_report() - - if not success: - raise AssertionError( - f"Benchmark failed for {self.model} with Eagle. Check the logs for details." - ) - - def test_eagle_accuracy_mgsm(self): - """Run MGSM accuracy evaluation for Mistral Large 3 with Eagle.""" - process = popen_launch_server( - model=self.model, - base_url=self.base_url, - other_args=self.other_args, - timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - ) - - try: - args = SimpleNamespace( - base_url=self.base_url, - model=self.model, - eval_name="mgsm_en", - num_examples=None, - num_threads=1024, - ) - metrics = run_eval(args) - print(f"MGSM accuracy for {self.model} with Eagle: {metrics['score']}") - - # Placeholder threshold - adjust after first successful run - expected_threshold = 0.90 - self.assertGreaterEqual( - metrics["score"], - expected_threshold, - f"MGSM accuracy {metrics['score']} below threshold {expected_threshold}", - ) - finally: - kill_process_tree(process.pid) - - -if __name__ == "__main__": - unittest.main() diff --git a/test/nightly/test_qwen3_235b_perf.py b/test/nightly/test_qwen3_235b_perf.py deleted file mode 100644 index 7988bbe7d..000000000 --- a/test/nightly/test_qwen3_235b_perf.py +++ /dev/null @@ -1,49 +0,0 @@ -import unittest - -from nightly_utils import NightlyBenchmarkRunner - -from sglang.test.test_utils import DEFAULT_URL_FOR_TEST, _parse_int_list_env - -QWEN3_235B_MODEL_PATH = "Qwen/Qwen3-235B-A22B-Instruct-2507" -PROFILE_DIR = "performance_profiles_qwen3_235b" - - -class TestNightlyQwen3235BPerformance(unittest.TestCase): - @classmethod - def setUpClass(cls): - cls.model = QWEN3_235B_MODEL_PATH - cls.base_url = DEFAULT_URL_FOR_TEST - cls.batch_sizes = [1, 1, 8, 16, 64] - cls.input_lens = tuple(_parse_int_list_env("NIGHTLY_INPUT_LENS", "4096")) - cls.output_lens = tuple(_parse_int_list_env("NIGHTLY_OUTPUT_LENS", "512")) - - # Qwen3-235B requires TP=8 for 8 GPUs - cls.other_args = [ - "--tp", - "8", - "--trust-remote-code", - ] - - cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url) - cls.runner.setup_profile_directory() - - def test_bench_one_batch(self): - results, success = self.runner.run_benchmark_for_model( - model_path=self.model, - batch_sizes=self.batch_sizes, - input_lens=self.input_lens, - output_lens=self.output_lens, - other_args=self.other_args, - ) - - self.runner.add_report(results) - self.runner.write_final_report() - - if not success: - raise AssertionError( - f"Benchmark failed for {self.model}. Check the logs for details." - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/test/registered/8-gpu-models/test_deepseek_v31.py b/test/registered/8-gpu-models/test_deepseek_v31.py new file mode 100644 index 000000000..a9816b11f --- /dev/null +++ b/test/registered/8-gpu-models/test_deepseek_v31.py @@ -0,0 +1,79 @@ +import sys +import unittest +from pathlib import Path + +# Add nightly directory to path for run_combined_tests import +sys.path.insert(0, str(Path(__file__).parent.parent.parent / "nightly")) + +from accuracy_test_runner import AccuracyTestParams +from performance_test_runner import PerformanceTestParams +from run_combined_tests import run_combined_tests + +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.test_utils import ModelLaunchSettings, is_blackwell_system + +# Runs on both H200 and B200 via nightly-8-gpu-common suite +register_cuda_ci(est_time=12000, suite="nightly-8-gpu-common", nightly=True) + +DEEPSEEK_V31_MODEL_PATH = "deepseek-ai/DeepSeek-V3.1" + + +@unittest.skipIf(not is_blackwell_system(), "Requires B200") +class TestDeepseekV31Unified(unittest.TestCase): + """Unified test class for DeepSeek-V3.1 performance and accuracy. + + Two variants: + - basic: Standard TP=8 + - mtp: TP=8 + EAGLE speculative decoding + + Each variant runs BOTH: + - Performance test (using NightlyBenchmarkRunner) + - Accuracy test (using run_eval with mgsm_en) + """ + + def test_deepseek_v31_all_variants(self): + """Run performance and accuracy for all DeepSeek-V3.1 variants.""" + # Define base arguments shared by most variants + base_args = [ + "--tp=8", + "--trust-remote-code", + "--model-loader-extra-config", + '{"enable_multithread_load": true}', + ] + mtp_args = [ + "--speculative-algorithm=EAGLE", + "--speculative-num-steps=3", + "--speculative-eagle-topk=1", + "--speculative-num-draft-tokens=4", + "--mem-frac=0.7", + ] + + variants = [ + # Variant: "basic" - Standard TP=8 + ModelLaunchSettings( + DEEPSEEK_V31_MODEL_PATH, + tp_size=8, + extra_args=base_args, + ), + # Variant: "mtp" - TP=8 + EAGLE speculative decoding + ModelLaunchSettings( + DEEPSEEK_V31_MODEL_PATH, + tp_size=8, + extra_args=base_args + mtp_args, + ), + ] + + run_combined_tests( + models=variants, + test_name="DeepSeek-V3.1 Unified", + accuracy_params=AccuracyTestParams( + dataset="gsm8k", baseline_accuracy=0.935 + ), + performance_params=PerformanceTestParams( + profile_dir="performance_profiles_deepseek_v31", + ), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/8-gpu-models/test_deepseek_v32.py b/test/registered/8-gpu-models/test_deepseek_v32.py new file mode 100644 index 000000000..e818939db --- /dev/null +++ b/test/registered/8-gpu-models/test_deepseek_v32.py @@ -0,0 +1,194 @@ +import sys +import unittest +from pathlib import Path + +# Add nightly directory to path for run_combined_tests import +sys.path.insert(0, str(Path(__file__).parent.parent.parent / "nightly")) + +from accuracy_test_runner import AccuracyTestParams +from performance_test_runner import PerformanceTestParams +from run_combined_tests import run_combined_tests + +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.test_utils import ModelLaunchSettings, is_blackwell_system + +register_cuda_ci(est_time=8000, suite="nightly-8-gpu-common", nightly=True) + +DEEPSEEK_V32_MODEL_PATH = "deepseek-ai/DeepSeek-V3.2" + +BASE_ARGS = [ + "--trust-remote-code", + "--model-loader-extra-config", + '{"enable_multithread_load": true}', +] + +DP_ARGS = [ + "--tp=8", + "--dp=8", + "--enable-dp-attention", +] + +# Accuracy thresholds +GSM8K_BASELINE = 0.935 +GPQA_BASELINE = 0.835 + + +class TestDeepseekV32Unified(unittest.TestCase): + """Unified test class for DeepSeek V3.2 performance and accuracy. + + Tests multiple variants with both performance and accuracy tests: + - dp: Standard TP=8 + DP=8 with dp-attention + - dp+mtp: DP + EAGLE speculative decoding + - tp: Pure TP=8 only + - tp+mtp: Pure TP=8 + EAGLE speculative decoding + """ + + @unittest.skipIf(is_blackwell_system(), "Requires H200 system") + def test_deepseek_v32_all_variants(self): + """Run performance and accuracy for all DeepSeek V3.2 variants.""" + TP_ARGS = [ + "--tp=8", + ] + MTP_ARGS = [ + "--speculative-algorithm=EAGLE", + "--speculative-num-steps=3", + "--speculative-eagle-topk=1", + "--speculative-num-draft-tokens=4", + "--mem-frac=0.7", + ] + variants = [ + # Variant: "dp" - Standard TP=8 + DP=8 with dp-attention + ModelLaunchSettings( + DEEPSEEK_V32_MODEL_PATH, + tp_size=8, + extra_args=BASE_ARGS + DP_ARGS, + ), + # Variant: "dp+mtp" - DP + EAGLE speculative decoding + ModelLaunchSettings( + DEEPSEEK_V32_MODEL_PATH, + tp_size=8, + extra_args=BASE_ARGS + DP_ARGS + MTP_ARGS, + ), + # Variant: "tp" - Pure TP=8 only + ModelLaunchSettings( + DEEPSEEK_V32_MODEL_PATH, + tp_size=8, + extra_args=BASE_ARGS + TP_ARGS, + ), + # Variant: "tp+mtp" - Pure TP=8 + EAGLE speculative decoding + ModelLaunchSettings( + DEEPSEEK_V32_MODEL_PATH, + tp_size=8, + extra_args=BASE_ARGS + TP_ARGS + MTP_ARGS, + ), + ] + + run_combined_tests( + models=variants, + test_name="DeepSeek-V3.2 Unified", + accuracy_params=AccuracyTestParams( + dataset="gsm8k", baseline_accuracy=GSM8K_BASELINE + ), + performance_params=PerformanceTestParams( + batch_sizes=[1, 8, 16, 64], + profile_dir="performance_profiles_deepseek_v32", + ), + ) + + @unittest.skipIf(is_blackwell_system(), "Requires H200 system") + def test_deepseek_v32_nsa_backends(self): + """Test NSA attention backend variants (H200 only). + + Tests three NSA backend configurations: + - flashmla: flashmla_sparse prefill + flashmla_kv decode + - fa3: FA3 prefill + FA3 decode + - fp8kvcache: default backends with FP8 KV cache + """ + NSA_FLASHMLA_ARGS = [ + "--attention-backend=nsa", + "--nsa-prefill-backend=flashmla_sparse", + "--nsa-decode-backend=flashmla_kv", + ] + + NSA_FA3_ARGS = [ + "--attention-backend=nsa", + "--nsa-prefill-backend=fa3", + "--nsa-decode-backend=fa3", + ] + + NSA_FP8KV_ARGS = [ + "--attention-backend=nsa", + "--kv-cache-dtype=fp8_e4m3", + ] + + nsa_variants = [ + # flashmla backend + ModelLaunchSettings( + DEEPSEEK_V32_MODEL_PATH, + tp_size=8, + extra_args=BASE_ARGS + DP_ARGS + NSA_FLASHMLA_ARGS, + ), + # fa3 backend + ModelLaunchSettings( + DEEPSEEK_V32_MODEL_PATH, + tp_size=8, + extra_args=BASE_ARGS + DP_ARGS + NSA_FA3_ARGS, + ), + # fp8 kv cache + ModelLaunchSettings( + DEEPSEEK_V32_MODEL_PATH, + tp_size=8, + extra_args=BASE_ARGS + DP_ARGS + NSA_FP8KV_ARGS, + ), + ] + + run_combined_tests( + models=nsa_variants, + test_name="DeepSeek-V3.2 NSA Backends", + accuracy_params=AccuracyTestParams( + dataset="gsm8k", baseline_accuracy=GSM8K_BASELINE + ), + performance_params=PerformanceTestParams( + batch_sizes=[1, 8, 16, 64], + profile_dir="performance_profiles_deepseek_v32_nsa", + ), + ) + + @unittest.skipIf(not is_blackwell_system(), "Requires B200") + def test_deepseek_v32_b200(self): + """Test DeepSeek V3.2 with GPQA evaluation using thinking mode (B200 only). + + This test runs GPQA evaluation with the reasoning parser enabled. + """ + B200_REASONING_ARGS = [ + "--tool-call-parser=deepseekv32", + "--reasoning-parser=deepseek-v3", + ] + + variants = [ + ModelLaunchSettings( + DEEPSEEK_V32_MODEL_PATH, + tp_size=8, + extra_args=BASE_ARGS + DP_ARGS + B200_REASONING_ARGS, + ), + ] + + run_combined_tests( + models=variants, + test_name="DeepSeek-V3.2 GPQA (B200)", + accuracy_params=AccuracyTestParams( + dataset="gpqa", + baseline_accuracy=GPQA_BASELINE, + num_examples=198, + num_threads=198, + max_tokens=120000, + thinking_mode="deepseek-v3", + temperature=0.1, + repeat=4, + ), + performance_params=None, # Skip performance test for GPQA + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/8-gpu-models/test_glm_46.py b/test/registered/8-gpu-models/test_glm_46.py new file mode 100644 index 000000000..dbd04d17e --- /dev/null +++ b/test/registered/8-gpu-models/test_glm_46.py @@ -0,0 +1,57 @@ +import sys +import unittest +from pathlib import Path + +# Add nightly directory to path for run_combined_tests import +sys.path.insert(0, str(Path(__file__).parent.parent.parent / "nightly")) + +from accuracy_test_runner import AccuracyTestParams +from performance_test_runner import PerformanceTestParams +from run_combined_tests import run_combined_tests + +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.test_utils import ModelLaunchSettings + +# Runs on both H200 and B200 via nightly-8-gpu-common suite +register_cuda_ci(est_time=12000, suite="nightly-8-gpu-common", nightly=True) + +GLM_4_6_MODEL_PATH = "zai-org/GLM-4.6" + + +class TestGLM46Unified(unittest.TestCase): + """Unified test class for GLM-4.6 performance and accuracy. + + Single variant with simple TP=8 configuration. + GLM-4.6 is a 357B MoE model. + Runs BOTH: + - Performance test (using NightlyBenchmarkRunner) + - Accuracy test (using run_eval with mgsm_en) + """ + + def test_glm_46(self): + """Run performance and accuracy for GLM-4.6.""" + base_args = [ + "--tp=8", + "--trust-remote-code", + ] + + variants = [ + ModelLaunchSettings( + GLM_4_6_MODEL_PATH, + tp_size=8, + extra_args=base_args, + ), + ] + + run_combined_tests( + models=variants, + test_name="GLM-4.6 Unified", + accuracy_params=AccuracyTestParams(dataset="gsm8k", baseline_accuracy=0.80), + performance_params=PerformanceTestParams( + profile_dir="performance_profiles_glm_4_6", + ), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/8-gpu-models/test_kimi_k2.py b/test/registered/8-gpu-models/test_kimi_k2.py new file mode 100644 index 000000000..5d3d9b05a --- /dev/null +++ b/test/registered/8-gpu-models/test_kimi_k2.py @@ -0,0 +1,58 @@ +import sys +import unittest +from pathlib import Path + +# Add nightly directory to path for run_combined_tests import +sys.path.insert(0, str(Path(__file__).parent.parent.parent / "nightly")) + +from accuracy_test_runner import AccuracyTestParams +from performance_test_runner import PerformanceTestParams +from run_combined_tests import run_combined_tests + +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.test_utils import ModelLaunchSettings + +# Runs on both H200 and B200 via nightly-8-gpu-common suite +register_cuda_ci(est_time=12000, suite="nightly-8-gpu-common", nightly=True) + +KIMI_K2_THINKING_MODEL_PATH = "moonshotai/Kimi-K2-Thinking" + + +class TestKimiK2Unified(unittest.TestCase): + """Unified test class for Kimi-K2-Thinking performance and accuracy. + + Single variant with TP=8 + tool/reasoning parsers. + Runs BOTH: + - Performance test (using NightlyBenchmarkRunner with extra_bench_args) + - Accuracy test (using run_eval with mgsm_en) + """ + + def test_kimi_k2(self): + """Run performance and accuracy for Kimi-K2-Thinking.""" + base_args = [ + "--tp=8", + "--trust-remote-code", + "--tool-call-parser=kimi_k2", + "--reasoning-parser=kimi_k2", + ] + + variants = [ + ModelLaunchSettings( + KIMI_K2_THINKING_MODEL_PATH, + tp_size=8, + extra_args=base_args, + ), + ] + + run_combined_tests( + models=variants, + test_name="Kimi-K2-Thinking Unified", + accuracy_params=AccuracyTestParams(dataset="gsm8k", baseline_accuracy=0.95), + performance_params=PerformanceTestParams( + profile_dir="performance_profiles_kimi_k2_thinking", + ), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/8-gpu-models/test_minimax_m2.py b/test/registered/8-gpu-models/test_minimax_m2.py new file mode 100644 index 000000000..b83ceff4d --- /dev/null +++ b/test/registered/8-gpu-models/test_minimax_m2.py @@ -0,0 +1,60 @@ +import sys +import unittest +from pathlib import Path + +# Add nightly directory to path for run_combined_tests import +sys.path.insert(0, str(Path(__file__).parent.parent.parent / "nightly")) + +from accuracy_test_runner import AccuracyTestParams +from performance_test_runner import PerformanceTestParams +from run_combined_tests import run_combined_tests + +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.test_utils import ModelLaunchSettings + +# Runs on both H200 and B200 via nightly-8-gpu-common suite +register_cuda_ci(est_time=12000, suite="nightly-8-gpu-common", nightly=True) + +MINIMAX_M2_MODEL_PATH = "MiniMaxAI/MiniMax-M2" + + +class TestMiniMaxM2Unified(unittest.TestCase): + """Unified test class for MiniMax-M2 performance and accuracy. + + Single variant with TP=8 + EP=8 configuration. + MiniMax-M2 is a 230B MoE model with 10B active params. + Runs BOTH: + - Performance test (using NightlyBenchmarkRunner with extra_bench_args) + - Accuracy test (using run_eval with mgsm_en) + """ + + def test_minimax_m2(self): + """Run performance and accuracy for MiniMax-M2.""" + base_args = [ + "--tp=8", + "--ep=8", + "--trust-remote-code", + "--model-loader-extra-config", + '{"enable_multithread_load": true}', + ] + + variants = [ + ModelLaunchSettings( + MINIMAX_M2_MODEL_PATH, + tp_size=8, + extra_args=base_args, + ), + ] + + run_combined_tests( + models=variants, + test_name="MiniMax-M2 Unified", + accuracy_params=AccuracyTestParams(dataset="gsm8k", baseline_accuracy=0.80), + performance_params=PerformanceTestParams( + profile_dir="performance_profiles_minimax_m2", + ), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/8-gpu-models/test_mistral_large3.py b/test/registered/8-gpu-models/test_mistral_large3.py new file mode 100644 index 000000000..aef06ba06 --- /dev/null +++ b/test/registered/8-gpu-models/test_mistral_large3.py @@ -0,0 +1,92 @@ +import os +import sys +import unittest +from pathlib import Path + +# Add nightly directory to path for run_combined_tests import +sys.path.insert(0, str(Path(__file__).parent.parent.parent / "nightly")) + +from accuracy_test_runner import AccuracyTestParams +from performance_test_runner import PerformanceTestParams +from run_combined_tests import run_combined_tests + +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.test_utils import ModelLaunchSettings, is_blackwell_system + +# Runs on both H200 and B200 via nightly-8-gpu-common suite +# Note: trtllm_mla backend may have hardware-specific behavior +register_cuda_ci(est_time=12000, suite="nightly-8-gpu-common", nightly=True) + +MISTRAL_LARGE3_MODEL_PATH = "mistralai/Mistral-Large-3-675B-Instruct-2512" +MISTRAL_LARGE3_EAGLE_MODEL_PATH = "mistralai/Mistral-Large-3-675B-Instruct-2512-Eagle" + + +@unittest.skipIf(not is_blackwell_system(), "Requires B200") +class TestMistralLarge3Unified(unittest.TestCase): + """Unified test class for Mistral-Large-3 performance and accuracy. + + Two variants: + - basic: TP=8 + trtllm_mla backend + - eagle: basic + EAGLE speculative decoding with draft model + + Each variant runs BOTH: + - Performance test (using NightlyBenchmarkRunner) + - Accuracy test (using run_eval with mgsm_en) + """ + + @classmethod + def setUpClass(cls): + # Set environment variable to disable JIT DeepGemm + os.environ["SGLANG_ENABLE_JIT_DEEPGEMM"] = "0" + + @classmethod + def tearDownClass(cls): + # Clean up environment variable + if "SGLANG_ENABLE_JIT_DEEPGEMM" in os.environ: + del os.environ["SGLANG_ENABLE_JIT_DEEPGEMM"] + + def test_mistral_large3_all_variants(self): + """Run performance and accuracy for all Mistral-Large-3 variants.""" + base_args = [ + "--tp=8", + "--attention-backend=trtllm_mla", + "--model-loader-extra-config", + '{"enable_multithread_load": true}', + "--chat-template=mistral", + ] + eagle_args = [ + "--speculative-algorithm=EAGLE", + f"--speculative-draft-model-path={MISTRAL_LARGE3_EAGLE_MODEL_PATH}", + "--speculative-num-steps=3", + "--speculative-eagle-topk=1", + "--speculative-num-draft-tokens=4", + "--kv-cache-dtype=auto", + ] + + variants = [ + # Variant: "basic" - TP=8 + trtllm_mla backend + ModelLaunchSettings( + MISTRAL_LARGE3_MODEL_PATH, + tp_size=8, + extra_args=base_args, + ), + # Variant: "eagle" - TP=8 + trtllm_mla + EAGLE with draft model + ModelLaunchSettings( + MISTRAL_LARGE3_MODEL_PATH, + tp_size=8, + extra_args=base_args + eagle_args, + ), + ] + + run_combined_tests( + models=variants, + test_name="Mistral-Large-3 Unified", + accuracy_params=AccuracyTestParams(dataset="gsm8k", baseline_accuracy=0.90), + performance_params=PerformanceTestParams( + profile_dir="performance_profiles_mistral_large3", + ), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/8-gpu-models/test_qwen3_235b.py b/test/registered/8-gpu-models/test_qwen3_235b.py new file mode 100644 index 000000000..17f5ec965 --- /dev/null +++ b/test/registered/8-gpu-models/test_qwen3_235b.py @@ -0,0 +1,57 @@ +import sys +import unittest +from pathlib import Path + +# Add nightly directory to path for run_combined_tests import +sys.path.insert(0, str(Path(__file__).parent.parent.parent / "nightly")) + +from accuracy_test_runner import AccuracyTestParams +from performance_test_runner import PerformanceTestParams +from run_combined_tests import run_combined_tests + +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.test_utils import ModelLaunchSettings, is_blackwell_system + +# Runs on both H200 and B200 via nightly-8-gpu-common suite +register_cuda_ci(est_time=12000, suite="nightly-8-gpu-common", nightly=True) + +QWEN3_235B_MODEL_PATH = "Qwen/Qwen3-235B-A22B-Instruct-2507" + + +@unittest.skipIf(not is_blackwell_system(), "Requires B200") +class TestQwen3235BUnified(unittest.TestCase): + """Unified test class for Qwen3-235B performance and accuracy. + + Single variant with simple TP=8 configuration. + Runs BOTH: + - Performance test (using NightlyBenchmarkRunner) + - Accuracy test (using run_eval with mgsm_en) + """ + + def test_qwen3_235b(self): + """Run performance and accuracy for Qwen3-235B.""" + base_args = [ + "--tp=8", + "--trust-remote-code", + ] + + variants = [ + ModelLaunchSettings( + QWEN3_235B_MODEL_PATH, + tp_size=8, + extra_args=base_args, + ), + ] + + run_combined_tests( + models=variants, + test_name="Qwen3-235B Unified", + accuracy_params=AccuracyTestParams(dataset="gsm8k", baseline_accuracy=0.88), + performance_params=PerformanceTestParams( + profile_dir="performance_profiles_qwen3_235b", + ), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/run_suite.py b/test/run_suite.py index 6f9b610d2..7203551e7 100644 --- a/test/run_suite.py +++ b/test/run_suite.py @@ -39,6 +39,9 @@ NIGHTLY_SUITES = { "nightly-8-gpu-h200", "nightly-8-gpu-h20", "nightly-8-gpu-b200", + "nightly-8-gpu-h200-basic", # Basic tests for large models on H200 + "nightly-8-gpu-b200-basic", # Basic tests for large models on B200 + "nightly-8-gpu-common", # Common tests that run on both H200 and B200 ], HWBackend.AMD: ["nightly-amd", "nightly-amd-8-gpu"], HWBackend.CPU: [],