diff --git a/.github/workflows/nightly-test-amd.yml b/.github/workflows/nightly-test-amd.yml index 155353975..936fbb778 100644 --- a/.github/workflows/nightly-test-amd.yml +++ b/.github/workflows/nightly-test-amd.yml @@ -22,6 +22,8 @@ on: # MI30x Accuracy Tests (GSM8K / MMMU) - 'nightly-accuracy-2-gpu' - 'nightly-accuracy-2-gpu-vlm' + - 'nightly-perf-2-gpu-text' + - 'nightly-perf-2-gpu-vlm' - 'nightly-accuracy-8-gpu' - 'nightly-accuracy-8-gpu-deepseek-r1' # MI30x Accuracy + Performance Tests (combined) @@ -31,10 +33,12 @@ on: # MI35x jobs - 'nightly-test-1-gpu-mi35x' - 'nightly-accuracy-8-gpu-mi35x' - - 'nightly-accuracy-8-gpu-mi35x-deepseek-r1' - 'nightly-8-gpu-mi35x-grok1-int4' - 'nightly-8-gpu-mi35x-grok2' - 'nightly-8-gpu-mi35x-deepseek-r1-mxfp4' + - 'nightly-accuracy-8-gpu-mi35x-deepseek-v32' + - 'nightly-perf-8-gpu-mi35x-deepseek-v32-basic' + - 'nightly-perf-8-gpu-mi35x-deepseek-v32-mtp' workflow_call: inputs: ref: @@ -106,6 +110,7 @@ jobs: - name: Nightly Test (2-GPU) run: | + > github_summary.md # Clear summary file bash scripts/ci/amd_ci_exec.sh -w /sglang-checkout/test \ -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ python3 run_suite.py --hw amd --suite nightly-amd --nightly --timeout-per-file 7200 || TEST_EXIT_CODE=$? @@ -135,12 +140,75 @@ jobs: - name: Nightly Accuracy Test (2-GPU VLM MMMU) timeout-minutes: 180 run: | + > github_summary.md # Clear summary file bash scripts/ci/amd_ci_exec.sh -w /sglang-checkout/test \ -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ python3 run_suite.py --hw amd --suite nightly-amd-accuracy-2-gpu-vlm --nightly --timeout-per-file 7200 || TEST_EXIT_CODE=$? echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} + # 2-GPU Text Models Performance Tests + nightly-perf-2-gpu-text: + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-perf-2-gpu-text') + runs-on: linux-mi325-gpu-2 + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref || github.ref }} + + - name: Setup docker + run: | + touch github_summary.md + bash scripts/ci/amd_ci_start_container.sh + env: + GITHUB_WORKSPACE: ${{ github.workspace }} + + - name: Install dependencies + run: bash scripts/ci/amd_ci_install_dependency.sh + + - name: Performance Test (2-GPU Text Models) + timeout-minutes: 120 + run: | + > github_summary.md # Clear summary file + bash scripts/ci/amd_ci_exec.sh -w /sglang-checkout/test \ + -e SGLANG_USE_AITER=1 \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + python3 run_suite.py --hw amd --suite nightly-amd-perf-text-2-gpu --nightly --timeout-per-file 3600 || TEST_EXIT_CODE=$? + echo "$(> $GITHUB_STEP_SUMMARY || true + exit ${TEST_EXIT_CODE:-0} + + # 2-GPU VLM Performance Tests + nightly-perf-2-gpu-vlm: + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-perf-2-gpu-vlm') + runs-on: linux-mi325-gpu-2 + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref || github.ref }} + + - name: Setup docker + run: | + touch github_summary.md + bash scripts/ci/amd_ci_start_container.sh + env: + GITHUB_WORKSPACE: ${{ github.workspace }} + + - name: Install dependencies + run: bash scripts/ci/amd_ci_install_dependency.sh + + - name: Performance Test (2-GPU VLM Models) + timeout-minutes: 180 + run: | + > github_summary.md # Clear summary file + bash scripts/ci/amd_ci_exec.sh -w /sglang-checkout/test \ + -e SGLANG_USE_AITER=1 \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + python3 run_suite.py --hw amd --suite nightly-amd-perf-vlm-2-gpu --nightly --timeout-per-file 7200 || TEST_EXIT_CODE=$? + echo "$(> $GITHUB_STEP_SUMMARY || true + exit ${TEST_EXIT_CODE:-0} + # 8-GPU Accuracy Tests - GPT-OSS, Grok1-FP8 (accuracy only) nightly-accuracy-8-gpu: if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-accuracy-8-gpu') @@ -404,38 +472,6 @@ jobs: echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} - # MI35x 8-GPU DeepSeek-R1-0528 Accuracy Test (separate job due to long loading time) - nightly-accuracy-8-gpu-mi35x-deepseek-r1: - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-accuracy-8-gpu-mi35x-deepseek-r1') - runs-on: linux-mi35x-gpu-8 - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref || github.ref }} - - - name: Setup docker - run: | - touch github_summary.md - bash scripts/ci/amd_ci_start_container.sh - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - - - name: Install dependencies - run: | - bash scripts/ci/amd_ci_install_dependency.sh - # Install tabulate for run_suite.py (missing in MI35x container) - bash scripts/ci/amd_ci_exec.sh pip install tabulate - - - name: Accuracy Test MI35x (8-GPU DeepSeek-R1-0528) - timeout-minutes: 240 - run: | - bash scripts/ci/amd_ci_exec.sh -w /sglang-checkout/test \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-mi35x-deepseek-r1 --nightly --timeout-per-file 7200 || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - # MI35x 8-GPU Grok1-INT4 (Accuracy + Performance combined) nightly-8-gpu-mi35x-grok1-int4: if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-8-gpu-mi35x-grok1-int4') @@ -572,6 +608,105 @@ jobs: echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} + # MI35x 8-GPU DeepSeek-V3.2 Accuracy Test + nightly-accuracy-8-gpu-mi35x-deepseek-v32: + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-accuracy-8-gpu-mi35x-deepseek-v32') + runs-on: linux-mi35x-gpu-8 + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref || github.ref }} + + - name: Setup docker + run: | + touch github_summary.md + bash scripts/ci/amd_ci_start_container.sh + env: + GITHUB_WORKSPACE: ${{ github.workspace }} + + - name: Install dependencies + run: | + bash scripts/ci/amd_ci_install_dependency.sh + # Install tabulate for run_suite.py (missing in MI35x container) + bash scripts/ci/amd_ci_exec.sh pip install tabulate + + - name: Accuracy Test MI35x (8-GPU DeepSeek-V3.2) + timeout-minutes: 120 + run: | + > github_summary.md # Clear summary file + bash scripts/ci/amd_ci_exec.sh -w /sglang-checkout/test \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x-deepseek-v32 --nightly --timeout-per-file 3600 || TEST_EXIT_CODE=$? + echo "$(> $GITHUB_STEP_SUMMARY || true + exit ${TEST_EXIT_CODE:-0} + + # MI35x 8-GPU DeepSeek-V3.2 Performance Test (Basic) + nightly-perf-8-gpu-mi35x-deepseek-v32-basic: + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-perf-8-gpu-mi35x-deepseek-v32-basic') + runs-on: linux-mi35x-gpu-8 + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref || github.ref }} + + - name: Setup docker + run: | + touch github_summary.md + bash scripts/ci/amd_ci_start_container.sh + env: + GITHUB_WORKSPACE: ${{ github.workspace }} + + - name: Install dependencies + run: | + bash scripts/ci/amd_ci_install_dependency.sh + # Install tabulate for run_suite.py (missing in MI35x container) + bash scripts/ci/amd_ci_exec.sh pip install tabulate + + - name: Performance Test MI35x (8-GPU DeepSeek-V3.2 Basic) + timeout-minutes: 150 + run: | + > github_summary.md # Clear summary file + bash scripts/ci/amd_ci_exec.sh -w /sglang-checkout/test \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-mi35x-deepseek-v32-basic --nightly --timeout-per-file 5400 || TEST_EXIT_CODE=$? + echo "$(> $GITHUB_STEP_SUMMARY || true + exit ${TEST_EXIT_CODE:-0} + + # MI35x 8-GPU DeepSeek-V3.2 Performance Test (MTP) + nightly-perf-8-gpu-mi35x-deepseek-v32-mtp: + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-perf-8-gpu-mi35x-deepseek-v32-mtp') + runs-on: linux-mi35x-gpu-8 + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref || github.ref }} + + - name: Setup docker + run: | + touch github_summary.md + bash scripts/ci/amd_ci_start_container.sh + env: + GITHUB_WORKSPACE: ${{ github.workspace }} + + - name: Install dependencies + run: | + bash scripts/ci/amd_ci_install_dependency.sh + # Install tabulate for run_suite.py (missing in MI35x container) + bash scripts/ci/amd_ci_exec.sh pip install tabulate + + - name: Performance Test MI35x (8-GPU DeepSeek-V3.2 MTP) + timeout-minutes: 150 + run: | + > github_summary.md # Clear summary file + bash scripts/ci/amd_ci_exec.sh -w /sglang-checkout/test \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-mi35x-deepseek-v32-mtp --nightly --timeout-per-file 5400 || TEST_EXIT_CODE=$? + echo "$(> $GITHUB_STEP_SUMMARY || true + exit ${TEST_EXIT_CODE:-0} + check-all-jobs: if: always() && (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch') needs: @@ -580,6 +715,9 @@ jobs: # MI30x Accuracy Tests - nightly-accuracy-2-gpu - nightly-accuracy-2-gpu-vlm + # MI30x Performance Tests + - nightly-perf-2-gpu-text + - nightly-perf-2-gpu-vlm - nightly-accuracy-8-gpu - nightly-accuracy-8-gpu-deepseek-r1 # MI30x Combined Accuracy + Performance Tests @@ -589,10 +727,12 @@ jobs: # MI35x jobs - nightly-test-1-gpu-mi35x - nightly-accuracy-8-gpu-mi35x - - nightly-accuracy-8-gpu-mi35x-deepseek-r1 - nightly-8-gpu-mi35x-grok1-int4 - nightly-8-gpu-mi35x-grok2 - nightly-8-gpu-mi35x-deepseek-r1-mxfp4 + - nightly-accuracy-8-gpu-mi35x-deepseek-v32 + - nightly-perf-8-gpu-mi35x-deepseek-v32-basic + - nightly-perf-8-gpu-mi35x-deepseek-v32-mtp runs-on: ubuntu-latest steps: - name: Check if any job failed diff --git a/scripts/ci/amd_ci_exec.sh b/scripts/ci/amd_ci_exec.sh index 84b1f347b..7c0ea9439 100755 --- a/scripts/ci/amd_ci_exec.sh +++ b/scripts/ci/amd_ci_exec.sh @@ -53,8 +53,11 @@ for key in "${!ENV_MAP[@]}"; do ENV_ARGS+=("-e" "$key=${ENV_MAP[$key]}") done -# Run docker exec with retry logic for HF network issues -# First attempt: normal mode +# Run docker exec with retry logic for HuggingFace network/download issues +# When HF model downloads fail due to network timeouts or rate limits, +# retrying with HF_HUB_OFFLINE=1 uses cached models from previous downloads. +# +# First attempt: normal mode (allows HF downloads) if docker exec \ -w "$WORKDIR" \ "${ENV_ARGS[@]}" \ @@ -65,7 +68,18 @@ else fi echo "First attempt failed with exit code $FIRST_EXIT_CODE" -echo "Retrying with HF_HUB_OFFLINE=1 (offline mode)..." + +# Skip retry for test failures that won't be fixed by offline mode: +# - Exit 1: Test assertion failures (accuracy below threshold) +# - Exit 137 (128+9): Process killed by OOM +# - Exit 255: Test suite completed with test errors +# Only retry for other exit codes (e.g., network timeouts, HF download failures) +if [[ "$FIRST_EXIT_CODE" -eq 1 || "$FIRST_EXIT_CODE" -eq 137 || "$FIRST_EXIT_CODE" -eq 255 ]]; then + echo "Exit code $FIRST_EXIT_CODE indicates test failure (not network issue), not retrying" + exit $FIRST_EXIT_CODE +fi + +echo "Retrying with HF_HUB_OFFLINE=1 (offline mode to use cached models)..." # Second attempt: force HF offline mode to avoid network timeouts docker exec \ diff --git a/test/registered/amd/accuracy/mi35x/test_deepseek_r1_mxfp4_eval_mi35x.py b/test/registered/amd/accuracy/mi35x/test_deepseek_r1_mxfp4_eval_mi35x.py index e7e8f6b9a..44491bfb8 100644 --- a/test/registered/amd/accuracy/mi35x/test_deepseek_r1_mxfp4_eval_mi35x.py +++ b/test/registered/amd/accuracy/mi35x/test_deepseek_r1_mxfp4_eval_mi35x.py @@ -1,7 +1,7 @@ """MI35x DeepSeek-R1-MXFP4 GSM8K Completion Evaluation Test (8-GPU) -Tests DeepSeek-R1-MXFP4 quantized model with multiple configurations -(basic, MTP, DP, TC) using few-shot completion benchmark on MI35x. +Tests DeepSeek-R1-MXFP4 quantized model with basic configuration +using few-shot completion benchmark on MI35x. Registry: nightly-amd-8-gpu-mi35x-deepseek-r1-mxfp4 suite """ @@ -32,9 +32,9 @@ from sglang.test.test_utils import ( ) from sglang.utils import download_and_cache_file, read_jsonl -# Register for AMD CI - MI35x DeepSeek-R1-MXFP4 accuracy tests (~120 min) +# Register for AMD CI - MI35x DeepSeek-R1-MXFP4 accuracy test (~60 min, basic only) register_amd_ci( - est_time=7200, suite="nightly-amd-8-gpu-mi35x-deepseek-r1-mxfp4", nightly=True + est_time=3600, suite="nightly-amd-8-gpu-mi35x-deepseek-r1-mxfp4", nightly=True ) INVALID = -9999999 @@ -83,7 +83,7 @@ def get_mxfp4_models() -> List[ModelConfig]: """Get DeepSeek-R1-MXFP4 model configurations for MI35x.""" model_path = get_model_path() return [ - # DeepSeek-R1-MXFP4 basic + # DeepSeek-R1-MXFP4 basic only (MTP tested in perf job) ModelConfig( model_path=model_path, tp_size=8, @@ -102,31 +102,6 @@ def get_mxfp4_models() -> List[ModelConfig]: ], env_vars={"SGLANG_USE_AITER": "1"}, ), - # DeepSeek-R1-MXFP4 with MTP (EAGLE) - ModelConfig( - model_path=model_path, - tp_size=8, - accuracy_threshold=0.93, - timeout=3600, - variant="MTP", - other_args=[ - "--chunked-prefill-size", - "131072", - "--speculative-algorithm", - "EAGLE", - "--speculative-num-steps", - "3", - "--speculative-eagle-topk", - "1", - "--speculative-num-draft-tokens", - "4", - "--mem-fraction-static", - "0.7", - "--trust-remote-code", - ], - env_vars={"SGLANG_USE_AITER": "1"}, - ), - # Note: DP and TC variants are not supported for MXFP4 on MI35x ] diff --git a/test/registered/amd/accuracy/mi35x/test_deepseek_v32_eval_mi35x.py b/test/registered/amd/accuracy/mi35x/test_deepseek_v32_eval_mi35x.py new file mode 100644 index 000000000..8861355a2 --- /dev/null +++ b/test/registered/amd/accuracy/mi35x/test_deepseek_v32_eval_mi35x.py @@ -0,0 +1,251 @@ +"""MI35x DeepSeek-V3.2 GSM8K Completion Evaluation Test (8-GPU) + +Tests DeepSeek-V3.2 with basic configuration using few-shot completion +benchmark on MI35x. + +Registry: nightly-amd-accuracy-8-gpu-mi35x-deepseek-v32 suite +""" + +import ast +import os + +# Set HF cache for MI35x +os.environ.setdefault("HF_HOME", "/data2/models/huggingface") +os.environ.setdefault("HF_HUB_CACHE", "/data2/models/huggingface/hub") + +import re +import time +import unittest +from dataclasses import dataclass +from typing import List, Optional, Tuple + +import numpy as np + +from sglang.srt.utils import kill_process_tree +from sglang.test.ci.ci_register import register_amd_ci +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + is_in_ci, + popen_launch_server, + write_github_step_summary, +) +from sglang.utils import download_and_cache_file, read_jsonl + +# Register for AMD CI - MI35x DeepSeek-V3.2 accuracy test (~60 min for basic only) +register_amd_ci( + est_time=3600, + suite="nightly-amd-8-gpu-mi35x-deepseek-v32", + nightly=True, +) + +INVALID = -9999999 + + +@dataclass +class ModelConfig: + """Configuration for a model to test.""" + + model_path: str + tp_size: int = 8 + accuracy_threshold: float = 0.50 + other_args: Optional[List[str]] = None + env_vars: Optional[dict] = None + timeout: Optional[int] = None + variant: Optional[str] = None + + def __post_init__(self): + if self.other_args is None: + self.other_args = [] + if self.env_vars is None: + self.env_vars = {} + + def get_display_name(self) -> str: + if self.variant: + return f"{self.model_path} ({self.variant})" + return self.model_path + + +# DeepSeek-V3.2 models for MI35x - only basic variant for nightly +# DP variant removed due to barrier deadlock during model loading +MI35X_DEEPSEEK_V32_MODELS = [ + # DeepSeek-V3.2 basic (TP=8 only) + ModelConfig( + model_path="deepseek-ai/DeepSeek-V3.2", + tp_size=8, + accuracy_threshold=0.93, + timeout=3600, + variant="basic", + other_args=[ + "--trust-remote-code", + "--nsa-prefill-backend", + "tilelang", + "--nsa-decode-backend", + "tilelang", + "--mem-fraction-static", + "0.85", + "--model-loader-extra-config", + '{"enable_multithread_load": true}', + "--watchdog-timeout", + "1200", # 20 minutes for weight loading + ], + env_vars={}, + ), +] + + +def get_one_example(lines, i, include_answer): + """Format a single GSM8K example.""" + ret = "Question: " + lines[i]["question"] + "\nAnswer:" + if include_answer: + ret += " " + lines[i]["answer"] + return ret + + +def get_few_shot_examples(lines, k): + """Get k few-shot examples for prompting.""" + ret = "" + for i in range(k): + ret += get_one_example(lines, i, True) + "\n\n" + return ret + + +def get_answer_value(answer_str): + """Extract numerical answer from response.""" + answer_str = answer_str.replace(",", "") + numbers = re.findall(r"\d+", answer_str) + if len(numbers) < 1: + return INVALID + try: + return ast.literal_eval(numbers[-1]) + except SyntaxError: + return INVALID + + +def run_gsm8k_benchmark( + base_url: str, + num_questions: int = 200, + num_shots: int = 5, + parallel: int = 64, +) -> Tuple[float, float, float]: + """Run GSM8K few-shot completion benchmark.""" + import sglang as sgl + from sglang.lang.backend.runtime_endpoint import RuntimeEndpoint + + url = "https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/test.jsonl" + data_path = download_and_cache_file(url) + lines = list(read_jsonl(data_path)) + + few_shot_examples = get_few_shot_examples(lines, num_shots) + + questions = [] + labels = [] + for i in range(len(lines[:num_questions])): + questions.append(get_one_example(lines, i, False)) + labels.append(get_answer_value(lines[i]["answer"])) + assert all(l != INVALID for l in labels) + arguments = [{"question": q} for q in questions] + + @sgl.function + def few_shot_gsm8k(s, question): + s += few_shot_examples + question + s += sgl.gen( + "answer", max_tokens=512, stop=["Question", "Assistant:", "<|separator|>"] + ) + + backend = RuntimeEndpoint(base_url) + sgl.set_default_backend(backend) + + tic = time.perf_counter() + states = few_shot_gsm8k.run_batch( + arguments, temperature=0, num_threads=parallel, progress_bar=True + ) + latency = time.perf_counter() - tic + + preds = [get_answer_value(states[i]["answer"]) for i in range(len(states))] + acc = np.mean(np.array(preds) == np.array(labels)) + invalid = np.mean(np.array(preds) == INVALID) + + return float(acc), float(invalid), float(latency) + + +class TestDeepSeekV32EvalMI35x(unittest.TestCase): + """DeepSeek-V3.2 GSM8K Completion Evaluation Test for AMD MI35x.""" + + @classmethod + def setUpClass(cls): + cls.models = MI35X_DEEPSEEK_V32_MODELS + cls.base_url = DEFAULT_URL_FOR_TEST + cls.num_questions = int(os.environ.get("GSM8K_NUM_QUESTIONS", "200")) + + def test_deepseek_v32_accuracy(self): + """Test DeepSeek-V3.2 models with GSM8K completion benchmark.""" + all_results = [] + summary = "### DeepSeek-V3.2 Models (MI35x)\n\n" + summary += "| Model | Variant | TP | Accuracy | Threshold | Status |\n" + summary += "| ----- | ------- | -- | -------- | --------- | ------ |\n" + + for config in self.models: + display_name = config.get_display_name() + with self.subTest(model=display_name): + print(f"\n{'='*60}") + print(f"Testing: {display_name}") + print(f"{'='*60}") + + env = os.environ.copy() + for key, value in config.env_vars.items(): + env[key] = value + + other_args = list(config.other_args) + other_args.extend(["--tp", str(config.tp_size)]) + timeout = config.timeout or DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH + + try: + process = popen_launch_server( + model=config.model_path, + base_url=self.base_url, + timeout=timeout, + other_args=other_args, + env=env, + ) + + try: + acc, invalid, latency = run_gsm8k_benchmark( + self.base_url, num_questions=self.num_questions + ) + passed = acc >= config.accuracy_threshold + status = "✅ PASS" if passed else "❌ FAIL" + + all_results.append( + { + "model": display_name, + "accuracy": acc, + "passed": passed, + } + ) + summary += f"| {config.model_path} | {config.variant or 'N/A'} | {config.tp_size} | {acc:.3f} | {config.accuracy_threshold} | {status} |\n" + + finally: + kill_process_tree(process.pid) + + except Exception as e: + summary += f"| {config.model_path} | {config.variant or 'N/A'} | {config.tp_size} | N/A | {config.accuracy_threshold} | ❌ ERROR |\n" + all_results.append( + { + "model": display_name, + "accuracy": None, + "passed": False, + "error": str(e), + } + ) + + if is_in_ci(): + write_github_step_summary(summary) + + failed = [r for r in all_results if not r["passed"]] + if failed: + raise AssertionError(f"Failed models: {[r['model'] for r in failed]}") + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/amd/nightly/test_gsm8k_eval_amd.py b/test/registered/amd/accuracy/test_gsm8k_eval_amd.py similarity index 100% rename from test/registered/amd/nightly/test_gsm8k_eval_amd.py rename to test/registered/amd/accuracy/test_gsm8k_eval_amd.py diff --git a/test/registered/amd/accuracy/test_vlms_mmmu_eval_amd.py b/test/registered/amd/accuracy/test_vlms_mmmu_eval_amd.py index 7909120a3..847f11b82 100644 --- a/test/registered/amd/accuracy/test_vlms_mmmu_eval_amd.py +++ b/test/registered/amd/accuracy/test_vlms_mmmu_eval_amd.py @@ -5,15 +5,18 @@ This test evaluates Vision-Language Models (VLMs) on the MMMU benchmark on AMD G Models are selected based on compatibility with AMD/ROCm platform. VLMs tested here: -- Qwen2-VL series (Qwen2-VL-7B, Qwen2.5-VL-7B) -- InternVL2 series -- MiniCPM-v series -- deepseek-vl2-small +- Qwen VL series (Qwen2-VL-7B, Qwen2.5-VL-7B, Qwen3-VL-30B) +- InternVL2 series (InternVL2_5-2B) +- MiniCPM series (MiniCPM-v-2_6, MiniCPM-o-2_6) +- DeepSeek VL series (deepseek-vl2-small, Janus-Pro-7B) +- Kimi VL (Kimi-VL-A3B-Instruct) +- MiMo VL (MiMo-VL-7B-RL) +- GLM VL (GLM-4.1V-9B-Thinking) -Note: Some VLMs from the Nvidia test are excluded due to AMD compatibility issues. +Note: NVILA models are excluded (NVIDIA-specific). Note: This test runs only on MI30x runners (linux-mi325-gpu-2), not on MI35x. -Registry: nightly-amd-vlm suite (2-GPU VLM tests) +Registry: nightly-amd-accuracy-2-gpu-vlm suite (2-GPU VLM tests) """ import os @@ -40,7 +43,7 @@ register_amd_ci(est_time=7200, suite="nightly-amd-accuracy-2-gpu-vlm", nightly=T # AMD-verified VLM models with conservative thresholds on 100 MMMU samples # Format: (model_path, tp_size, accuracy_threshold, extra_args) AMD_VLM_MODELS = [ - # Qwen2-VL series - well supported on AMD + # Qwen VL series - well supported on AMD { "model_path": "Qwen/Qwen2-VL-7B-Instruct", "tp_size": 1, @@ -53,6 +56,12 @@ AMD_VLM_MODELS = [ "accuracy_threshold": 0.33, "extra_args": ["--trust-remote-code"], }, + { + "model_path": "Qwen/Qwen3-VL-30B-A3B-Instruct", + "tp_size": 2, + "accuracy_threshold": 0.29, + "extra_args": ["--trust-remote-code"], + }, # InternVL2 - smaller model, good for testing { "model_path": "OpenGVLab/InternVL2_5-2B", @@ -60,25 +69,60 @@ AMD_VLM_MODELS = [ "accuracy_threshold": 0.29, "extra_args": ["--trust-remote-code"], }, - # MiniCPM-v - lightweight VLM + # MiniCPM series { "model_path": "openbmb/MiniCPM-v-2_6", "tp_size": 1, "accuracy_threshold": 0.25, "extra_args": ["--trust-remote-code"], }, - # DeepSeek VL2 small - MoE VLM + { + "model_path": "openbmb/MiniCPM-o-2_6", + "tp_size": 1, + "accuracy_threshold": 0.32, + "extra_args": ["--trust-remote-code"], + }, + # DeepSeek VL series { "model_path": "deepseek-ai/deepseek-vl2-small", "tp_size": 1, "accuracy_threshold": 0.31, "extra_args": ["--trust-remote-code"], }, + { + "model_path": "deepseek-ai/Janus-Pro-7B", + "tp_size": 1, + "accuracy_threshold": 0.28, + "extra_args": ["--trust-remote-code"], + }, + # Kimi VL - MoE + { + "model_path": "moonshotai/Kimi-VL-A3B-Instruct", + "tp_size": 1, + "accuracy_threshold": 0.26, + "extra_args": ["--trust-remote-code"], + }, + # MiMo VL + { + "model_path": "XiaomiMiMo/MiMo-VL-7B-RL", + "tp_size": 1, + "accuracy_threshold": 0.27, + "extra_args": ["--trust-remote-code"], + }, + # GLM VL + { + "model_path": "zai-org/GLM-4.1V-9B-Thinking", + "tp_size": 1, + "accuracy_threshold": 0.27, + "extra_args": ["--trust-remote-code"], + }, ] -# Models that need special handling on AMD +# Models that need special handling on AMD (MoE models) TRITON_ATTENTION_MODELS = { - "deepseek-ai/deepseek-vl2-small", # MoE model + "deepseek-ai/deepseek-vl2-small", + "Qwen/Qwen3-VL-30B-A3B-Instruct", + "moonshotai/Kimi-VL-A3B-Instruct", } # Models known to fail on AMD - exclude from testing diff --git a/test/registered/amd/perf/mi35x/test_deepseek_r1_mxfp4_perf_mi35x.py b/test/registered/amd/perf/mi35x/test_deepseek_r1_mxfp4_perf_mi35x.py index 7660d0b6e..01be06ebd 100644 --- a/test/registered/amd/perf/mi35x/test_deepseek_r1_mxfp4_perf_mi35x.py +++ b/test/registered/amd/perf/mi35x/test_deepseek_r1_mxfp4_perf_mi35x.py @@ -30,7 +30,10 @@ register_amd_ci( def generate_simple_markdown_report(results: List[BenchmarkResult]) -> str: - """Generate a simplified markdown report without traces and cost columns.""" + """Generate a simplified markdown report without traces and cost columns. + + Skips the first result if it's a warmup run (duplicate batch_size). + """ model_header = results[0].model_path if results[0].run_name and results[0].run_name != "default": model_header += f" ({results[0].run_name})" @@ -43,7 +46,14 @@ def generate_simple_markdown_report(results: List[BenchmarkResult]) -> str: summary += "| batch size | input len | latency (s) | input throughput (tok/s) | output throughput (tok/s) | ITL (ms) |\n" summary += "| ---------- | --------- | ----------- | ------------------------ | ------------------------- | -------- |\n" - for result in results: + # Skip first result if it's a warmup (same batch_size as second result) + report_results = ( + results[1:] + if len(results) > 1 and results[0].batch_size == results[1].batch_size + else results + ) + + for result in report_results: itl = 1 / (result.output_throughput / result.batch_size) * 1000 summary += f"| {result.batch_size} | {result.input_len} | {result.latency:.2f} | {result.input_throughput:.2f} | {result.output_throughput:.2f} | {itl:.2f} |\n" @@ -82,7 +92,7 @@ class TestDeepseekR1MXFP4PerfMI35x(unittest.TestCase): cls.model = get_model_path() print(f"Using model path: {cls.model}") cls.base_url = DEFAULT_URL_FOR_TEST - cls.batch_sizes = [1, 1, 8, 16, 64] + cls.batch_sizes = [1, 8, 16, 64] cls.input_lens = tuple(_parse_int_list_env("NIGHTLY_INPUT_LENS", "4096")) cls.output_lens = tuple(_parse_int_list_env("NIGHTLY_OUTPUT_LENS", "512")) diff --git a/test/registered/amd/perf/mi35x/test_deepseek_v32_basic_perf_mi35x.py b/test/registered/amd/perf/mi35x/test_deepseek_v32_basic_perf_mi35x.py new file mode 100644 index 000000000..96365d9c7 --- /dev/null +++ b/test/registered/amd/perf/mi35x/test_deepseek_v32_basic_perf_mi35x.py @@ -0,0 +1,134 @@ +"""MI35x Nightly performance benchmark for DeepSeek-V3.2 model (basic variant). + +This test benchmarks the DeepSeek-V3.2 model with basic TP=8 configuration on 8 GPUs. + +The model path can be configured via DEEPSEEK_V32_MODEL_PATH environment variable. + +Registry: nightly-perf-8-gpu-mi35x-deepseek-v32-basic suite + +Example usage: + DEEPSEEK_V32_MODEL_PATH=deepseek-ai/DeepSeek-V3.2 python -m pytest test_deepseek_v32_basic_perf_mi35x.py -v +""" + +import os +import unittest +from typing import List + +from sglang.test.ci.ci_register import register_amd_ci +from sglang.test.nightly_bench_utils import BenchmarkResult +from sglang.test.nightly_utils import NightlyBenchmarkRunner +from sglang.test.test_utils import DEFAULT_URL_FOR_TEST, _parse_int_list_env + +# Register for AMD CI - DeepSeek-V3.2 basic benchmark (~90 min) +register_amd_ci( + est_time=5400, suite="nightly-perf-8-gpu-mi35x-deepseek-v32-basic", nightly=True +) + + +def generate_simple_markdown_report(results: List[BenchmarkResult]) -> str: + """Generate a simplified markdown report without traces and cost columns. + + Skips the first result if it's a warmup run (duplicate batch_size). + """ + model_header = results[0].model_path + if results[0].run_name and results[0].run_name != "default": + model_header += f" ({results[0].run_name})" + + gpu_config = os.getenv("GPU_CONFIG", "MI35x") + if gpu_config: + model_header += f" [{gpu_config}]" + + summary = f"### {model_header}\n" + summary += "| batch size | input len | latency (s) | input throughput (tok/s) | output throughput (tok/s) | ITL (ms) |\n" + summary += "| ---------- | --------- | ----------- | ------------------------ | ------------------------- | -------- |\n" + + # Skip first result if it's a warmup (same batch_size as second result) + report_results = ( + results[1:] + if len(results) > 1 and results[0].batch_size == results[1].batch_size + else results + ) + + for result in report_results: + itl = 1 / (result.output_throughput / result.batch_size) * 1000 + summary += f"| {result.batch_size} | {result.input_len} | {result.latency:.2f} | {result.input_throughput:.2f} | {result.output_throughput:.2f} | {itl:.2f} |\n" + + return summary + + +# Model path can be overridden via environment variable +DEEPSEEK_V32_MODEL_PATH = os.environ.get( + "DEEPSEEK_V32_MODEL_PATH", "deepseek-ai/DeepSeek-V3.2" +) +PROFILE_DIR = "performance_profiles_deepseek_v32_basic" + + +class TestNightlyDeepseekV32BasicPerformance(unittest.TestCase): + """MI35x Nightly performance benchmark for DeepSeek-V3.2 model (basic variant). + + Tests the DeepSeek-V3.2 model with basic TP=8 configuration. + """ + + @classmethod + def setUpClass(cls): + cls.model = DEEPSEEK_V32_MODEL_PATH + cls.base_url = DEFAULT_URL_FOR_TEST + cls.batch_sizes = [1, 8, 16, 64] + cls.input_lens = tuple(_parse_int_list_env("NIGHTLY_INPUT_LENS", "4096")) + cls.output_lens = tuple(_parse_int_list_env("NIGHTLY_OUTPUT_LENS", "512")) + + # Basic variant configuration for DeepSeek-V3.2 + # MI35x uses tilelang NSA backends + cls.variant_config = { + "name": "basic", + "other_args": [ + "--trust-remote-code", + "--tp", + "8", + "--nsa-prefill-backend", + "tilelang", + "--nsa-decode-backend", + "tilelang", + "--mem-fraction-static", + "0.85", + "--model-loader-extra-config", + '{"enable_multithread_load": true}', + ], + } + + cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url) + cls.runner.setup_profile_directory() + # Override full_report to remove traces help text + cls.runner.full_report = f"## {cls.__name__}\n" + + def test_bench_one_batch(self): + """Run benchmark for basic variant.""" + try: + result_tuple = self.runner.run_benchmark_for_model( + model_path=self.model, + batch_sizes=self.batch_sizes, + input_lens=self.input_lens, + output_lens=self.output_lens, + other_args=self.variant_config["other_args"], + variant=self.variant_config["name"], + extra_bench_args=["--trust-remote-code"], + ) + results = result_tuple[0] + success = result_tuple[1] + + # Use simplified report format without traces + if results: + self.runner.full_report += ( + generate_simple_markdown_report(results) + "\n" + ) + + if not success: + raise AssertionError( + f"Benchmark failed for {self.model} (basic variant)" + ) + finally: + self.runner.write_final_report() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/amd/perf/mi35x/test_deepseek_v32_mtp_perf_mi35x.py b/test/registered/amd/perf/mi35x/test_deepseek_v32_mtp_perf_mi35x.py new file mode 100644 index 000000000..01aa19bb7 --- /dev/null +++ b/test/registered/amd/perf/mi35x/test_deepseek_v32_mtp_perf_mi35x.py @@ -0,0 +1,146 @@ +"""MI35x Nightly performance benchmark for DeepSeek-V3.2 model (MTP variant). + +This test benchmarks the DeepSeek-V3.2 model with MTP (EAGLE speculative decoding) +configuration on 8 GPUs. + +The model path can be configured via DEEPSEEK_V32_MODEL_PATH environment variable. + +Registry: nightly-perf-8-gpu-mi35x-deepseek-v32-mtp suite + +Example usage: + DEEPSEEK_V32_MODEL_PATH=deepseek-ai/DeepSeek-V3.2 python -m pytest test_deepseek_v32_mtp_perf_mi35x.py -v +""" + +import os +import unittest +from typing import List + +from sglang.test.ci.ci_register import register_amd_ci +from sglang.test.nightly_bench_utils import BenchmarkResult +from sglang.test.nightly_utils import NightlyBenchmarkRunner +from sglang.test.test_utils import DEFAULT_URL_FOR_TEST, _parse_int_list_env + +# Register for AMD CI - DeepSeek-V3.2 MTP benchmark (~90 min) +register_amd_ci( + est_time=5400, suite="nightly-perf-8-gpu-mi35x-deepseek-v32-mtp", nightly=True +) + + +def generate_simple_markdown_report(results: List[BenchmarkResult]) -> str: + """Generate a simplified markdown report without traces and cost columns. + + Skips the first result if it's a warmup run (duplicate batch_size). + """ + model_header = results[0].model_path + if results[0].run_name and results[0].run_name != "default": + model_header += f" ({results[0].run_name})" + + gpu_config = os.getenv("GPU_CONFIG", "MI35x") + if gpu_config: + model_header += f" [{gpu_config}]" + + summary = f"### {model_header}\n" + summary += "| batch size | input len | latency (s) | input throughput (tok/s) | output throughput (tok/s) | ITL (ms) |\n" + summary += "| ---------- | --------- | ----------- | ------------------------ | ------------------------- | -------- |\n" + + # Skip first result if it's a warmup (same batch_size as second result) + report_results = ( + results[1:] + if len(results) > 1 and results[0].batch_size == results[1].batch_size + else results + ) + + for result in report_results: + itl = 1 / (result.output_throughput / result.batch_size) * 1000 + summary += f"| {result.batch_size} | {result.input_len} | {result.latency:.2f} | {result.input_throughput:.2f} | {result.output_throughput:.2f} | {itl:.2f} |\n" + + return summary + + +# Model path can be overridden via environment variable +DEEPSEEK_V32_MODEL_PATH = os.environ.get( + "DEEPSEEK_V32_MODEL_PATH", "deepseek-ai/DeepSeek-V3.2" +) +PROFILE_DIR = "performance_profiles_deepseek_v32_mtp" + + +class TestNightlyDeepseekV32MTPPerformance(unittest.TestCase): + """MI35x Nightly performance benchmark for DeepSeek-V3.2 model (MTP variant). + + Tests the DeepSeek-V3.2 model with MTP (EAGLE speculative decoding) on TP=8. + """ + + @classmethod + def setUpClass(cls): + cls.model = DEEPSEEK_V32_MODEL_PATH + cls.base_url = DEFAULT_URL_FOR_TEST + cls.batch_sizes = [1, 8, 16, 64] + cls.input_lens = tuple(_parse_int_list_env("NIGHTLY_INPUT_LENS", "4096")) + cls.output_lens = tuple(_parse_int_list_env("NIGHTLY_OUTPUT_LENS", "512")) + + # MTP variant configuration for DeepSeek-V3.2 + # MI35x uses tilelang NSA backends + EAGLE speculative decoding + cls.variant_config = { + "name": "mtp", + "other_args": [ + "--trust-remote-code", + "--tp", + "8", + "--nsa-prefill-backend", + "tilelang", + "--nsa-decode-backend", + "tilelang", + "--speculative-algorithm", + "EAGLE", + "--speculative-num-steps", + "3", + "--speculative-eagle-topk", + "1", + "--speculative-num-draft-tokens", + "4", + "--mem-fraction-static", + "0.7", + "--model-loader-extra-config", + '{"enable_multithread_load": true}', + ], + } + + cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url) + cls.runner.setup_profile_directory() + # Override full_report to remove traces help text + cls.runner.full_report = f"## {cls.__name__}\n" + + def test_bench_one_batch(self): + """Run benchmark for MTP variant.""" + try: + result_tuple = self.runner.run_benchmark_for_model( + model_path=self.model, + batch_sizes=self.batch_sizes, + input_lens=self.input_lens, + output_lens=self.output_lens, + other_args=self.variant_config["other_args"], + variant=self.variant_config["name"], + extra_bench_args=["--trust-remote-code"], + ) + results = result_tuple[0] + success = result_tuple[1] + avg_spec_accept_length = result_tuple[2] if len(result_tuple) > 2 else None + + # Log speculative decoding accept length + if avg_spec_accept_length is not None: + print(f" avg_spec_accept_length={avg_spec_accept_length:.2f}") + + # Use simplified report format without traces + if results: + self.runner.full_report += ( + generate_simple_markdown_report(results) + "\n" + ) + + if not success: + raise AssertionError(f"Benchmark failed for {self.model} (MTP variant)") + finally: + self.runner.write_final_report() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/amd/perf/mi35x/test_grok1_int4_perf_mi35x.py b/test/registered/amd/perf/mi35x/test_grok1_int4_perf_mi35x.py index 5ffd8d000..489d62eda 100644 --- a/test/registered/amd/perf/mi35x/test_grok1_int4_perf_mi35x.py +++ b/test/registered/amd/perf/mi35x/test_grok1_int4_perf_mi35x.py @@ -21,7 +21,10 @@ register_amd_ci( def generate_simple_markdown_report(results: List[BenchmarkResult]) -> str: - """Generate a simplified markdown report without traces and cost columns.""" + """Generate a simplified markdown report without traces and cost columns. + + Skips the first result if it's a warmup run (duplicate batch_size). + """ model_header = results[0].model_path if results[0].run_name and results[0].run_name != "default": model_header += f" ({results[0].run_name})" @@ -34,7 +37,14 @@ def generate_simple_markdown_report(results: List[BenchmarkResult]) -> str: summary += "| batch size | input len | latency (s) | input throughput (tok/s) | output throughput (tok/s) | ITL (ms) |\n" summary += "| ---------- | --------- | ----------- | ------------------------ | ------------------------- | -------- |\n" - for result in results: + # Skip first result if it's a warmup (same batch_size as second result) + report_results = ( + results[1:] + if len(results) > 1 and results[0].batch_size == results[1].batch_size + else results + ) + + for result in report_results: itl = 1 / (result.output_throughput / result.batch_size) * 1000 summary += f"| {result.batch_size} | {result.input_len} | {result.latency:.2f} | {result.input_throughput:.2f} | {result.output_throughput:.2f} | {itl:.2f} |\n" @@ -53,7 +63,7 @@ class TestGrok1INT4PerfMI35x(unittest.TestCase): @classmethod def setUpClass(cls): cls.base_url = DEFAULT_URL_FOR_TEST - cls.batch_sizes = [1, 1, 8, 16, 64] + cls.batch_sizes = [1, 8, 16, 64] cls.input_lens = tuple(_parse_int_list_env("NIGHTLY_INPUT_LENS", "1024")) cls.output_lens = tuple(_parse_int_list_env("NIGHTLY_OUTPUT_LENS", "512")) diff --git a/test/registered/amd/perf/mi35x/test_grok2_perf_mi35x.py b/test/registered/amd/perf/mi35x/test_grok2_perf_mi35x.py index ba55d1e5b..8e3ba7231 100644 --- a/test/registered/amd/perf/mi35x/test_grok2_perf_mi35x.py +++ b/test/registered/amd/perf/mi35x/test_grok2_perf_mi35x.py @@ -19,7 +19,10 @@ register_amd_ci(est_time=1500, suite="nightly-perf-8-gpu-mi35x-grok2", nightly=T def generate_simple_markdown_report(results: List[BenchmarkResult]) -> str: - """Generate a simplified markdown report without traces and cost columns.""" + """Generate a simplified markdown report without traces and cost columns. + + Skips the first result if it's a warmup run (duplicate batch_size). + """ model_header = results[0].model_path if results[0].run_name and results[0].run_name != "default": model_header += f" ({results[0].run_name})" @@ -32,7 +35,14 @@ def generate_simple_markdown_report(results: List[BenchmarkResult]) -> str: summary += "| batch size | input len | latency (s) | input throughput (tok/s) | output throughput (tok/s) | ITL (ms) |\n" summary += "| ---------- | --------- | ----------- | ------------------------ | ------------------------- | -------- |\n" - for result in results: + # Skip first result if it's a warmup (same batch_size as second result) + report_results = ( + results[1:] + if len(results) > 1 and results[0].batch_size == results[1].batch_size + else results + ) + + for result in report_results: itl = 1 / (result.output_throughput / result.batch_size) * 1000 summary += f"| {result.batch_size} | {result.input_len} | {result.latency:.2f} | {result.input_throughput:.2f} | {result.output_throughput:.2f} | {itl:.2f} |\n" @@ -53,7 +63,7 @@ class TestGrok2PerfMI35x(unittest.TestCase): @classmethod def setUpClass(cls): cls.base_url = DEFAULT_URL_FOR_TEST - cls.batch_sizes = [1, 1, 8, 16, 64] + cls.batch_sizes = [1, 8, 16, 64] cls.input_lens = tuple(_parse_int_list_env("NIGHTLY_INPUT_LENS", "1024")) cls.output_lens = tuple(_parse_int_list_env("NIGHTLY_OUTPUT_LENS", "512")) diff --git a/test/registered/amd/perf/test_deepseek_v31_perf.py b/test/registered/amd/perf/test_deepseek_v31_perf.py index 91d5b1004..5c8c50d99 100644 --- a/test/registered/amd/perf/test_deepseek_v31_perf.py +++ b/test/registered/amd/perf/test_deepseek_v31_perf.py @@ -22,7 +22,10 @@ register_amd_ci(est_time=18000, suite="nightly-perf-8-gpu-deepseek-v31", nightly def generate_simple_markdown_report(results: List[BenchmarkResult]) -> str: - """Generate a simplified markdown report without traces and cost columns.""" + """Generate a simplified markdown report without traces and cost columns. + + Skips the first result if it's a warmup run (duplicate batch_size). + """ model_header = results[0].model_path if results[0].run_name and results[0].run_name != "default": model_header += f" ({results[0].run_name})" @@ -35,7 +38,14 @@ def generate_simple_markdown_report(results: List[BenchmarkResult]) -> str: summary += "| batch size | input len | latency (s) | input throughput (tok/s) | output throughput (tok/s) | ITL (ms) |\n" summary += "| ---------- | --------- | ----------- | ------------------------ | ------------------------- | -------- |\n" - for result in results: + # Skip first result if it's a warmup (same batch_size as second result) + report_results = ( + results[1:] + if len(results) > 1 and results[0].batch_size == results[1].batch_size + else results + ) + + for result in report_results: itl = 1 / (result.output_throughput / result.batch_size) * 1000 summary += f"| {result.batch_size} | {result.input_len} | {result.latency:.2f} | {result.input_throughput:.2f} | {result.output_throughput:.2f} | {itl:.2f} |\n" @@ -59,7 +69,7 @@ class TestNightlyDeepseekV31Performance(unittest.TestCase): def setUpClass(cls): cls.model = DEEPSEEK_V31_MODEL_PATH cls.base_url = DEFAULT_URL_FOR_TEST - cls.batch_sizes = [1, 1, 8, 16, 64] + cls.batch_sizes = [1, 8, 16, 64] cls.input_lens = tuple(_parse_int_list_env("NIGHTLY_INPUT_LENS", "4096")) cls.output_lens = tuple(_parse_int_list_env("NIGHTLY_OUTPUT_LENS", "512")) diff --git a/test/registered/amd/perf/test_grok1_int4_perf.py b/test/registered/amd/perf/test_grok1_int4_perf.py index ed336e056..a2164a8f9 100644 --- a/test/registered/amd/perf/test_grok1_int4_perf.py +++ b/test/registered/amd/perf/test_grok1_int4_perf.py @@ -24,7 +24,10 @@ register_amd_ci(est_time=1500, suite="nightly-perf-8-gpu-grok1-int4", nightly=Tr def generate_simple_markdown_report(results: List[BenchmarkResult]) -> str: - """Generate a simplified markdown report without traces and cost columns.""" + """Generate a simplified markdown report without traces and cost columns. + + Skips the first result if it's a warmup run (duplicate batch_size). + """ model_header = results[0].model_path if results[0].run_name and results[0].run_name != "default": model_header += f" ({results[0].run_name})" @@ -37,7 +40,14 @@ def generate_simple_markdown_report(results: List[BenchmarkResult]) -> str: summary += "| batch size | input len | latency (s) | input throughput (tok/s) | output throughput (tok/s) | ITL (ms) |\n" summary += "| ---------- | --------- | ----------- | ------------------------ | ------------------------- | -------- |\n" - for result in results: + # Skip first result if it's a warmup (same batch_size as second result) + report_results = ( + results[1:] + if len(results) > 1 and results[0].batch_size == results[1].batch_size + else results + ) + + for result in report_results: itl = 1 / (result.output_throughput / result.batch_size) * 1000 summary += f"| {result.batch_size} | {result.input_len} | {result.latency:.2f} | {result.input_throughput:.2f} | {result.output_throughput:.2f} | {itl:.2f} |\n" @@ -60,7 +70,7 @@ class TestNightlyGrok1INT4Performance(unittest.TestCase): @classmethod def setUpClass(cls): cls.base_url = DEFAULT_URL_FOR_TEST - cls.batch_sizes = [1, 1, 8, 16, 64] + cls.batch_sizes = [1, 8, 16, 64] cls.input_lens = tuple(_parse_int_list_env("NIGHTLY_INPUT_LENS", "1024")) cls.output_lens = tuple(_parse_int_list_env("NIGHTLY_OUTPUT_LENS", "512")) diff --git a/test/registered/amd/perf/test_grok2_perf.py b/test/registered/amd/perf/test_grok2_perf.py index ed163e2f3..e5b66b782 100644 --- a/test/registered/amd/perf/test_grok2_perf.py +++ b/test/registered/amd/perf/test_grok2_perf.py @@ -24,7 +24,10 @@ register_amd_ci(est_time=1500, suite="nightly-perf-8-gpu-grok2", nightly=True) def generate_simple_markdown_report(results: List[BenchmarkResult]) -> str: - """Generate a simplified markdown report without traces and cost columns.""" + """Generate a simplified markdown report without traces and cost columns. + + Skips the first result if it's a warmup run (duplicate batch_size). + """ model_header = results[0].model_path if results[0].run_name and results[0].run_name != "default": model_header += f" ({results[0].run_name})" @@ -37,7 +40,14 @@ def generate_simple_markdown_report(results: List[BenchmarkResult]) -> str: summary += "| batch size | input len | latency (s) | input throughput (tok/s) | output throughput (tok/s) | ITL (ms) |\n" summary += "| ---------- | --------- | ----------- | ------------------------ | ------------------------- | -------- |\n" - for result in results: + # Skip first result if it's a warmup (same batch_size as second result) + report_results = ( + results[1:] + if len(results) > 1 and results[0].batch_size == results[1].batch_size + else results + ) + + for result in report_results: itl = 1 / (result.output_throughput / result.batch_size) * 1000 summary += f"| {result.batch_size} | {result.input_len} | {result.latency:.2f} | {result.input_throughput:.2f} | {result.output_throughput:.2f} | {itl:.2f} |\n" @@ -62,7 +72,7 @@ class TestNightlyGrok2Performance(unittest.TestCase): @classmethod def setUpClass(cls): cls.base_url = DEFAULT_URL_FOR_TEST - cls.batch_sizes = [1, 1, 8, 16, 64] + cls.batch_sizes = [1, 8, 16, 64] cls.input_lens = tuple(_parse_int_list_env("NIGHTLY_INPUT_LENS", "1024")) cls.output_lens = tuple(_parse_int_list_env("NIGHTLY_OUTPUT_LENS", "512")) diff --git a/test/registered/amd/perf/test_text_models_perf_amd.py b/test/registered/amd/perf/test_text_models_perf_amd.py new file mode 100644 index 000000000..d03788ee2 --- /dev/null +++ b/test/registered/amd/perf/test_text_models_perf_amd.py @@ -0,0 +1,132 @@ +"""AMD Nightly performance benchmark for text models (2-GPU). + +This test benchmarks text models on AMD MI30x/MI35x with 2 GPUs. + +Registry: nightly-amd-perf-text-2-gpu suite + +Example usage: + python -m pytest test_text_models_perf_amd.py -v +""" + +import os +import unittest +from typing import List + +from sglang.test.ci.ci_register import register_amd_ci +from sglang.test.nightly_bench_utils import BenchmarkResult +from sglang.test.nightly_utils import NightlyBenchmarkRunner +from sglang.test.test_utils import ( + DEFAULT_URL_FOR_TEST, + ModelLaunchSettings, + _parse_int_list_env, + parse_models, +) + +# Register for AMD CI - Text models benchmark (~60 min) +register_amd_ci(est_time=3600, suite="nightly-amd-perf-text-2-gpu", nightly=True) + +PROFILE_DIR = "performance_profiles_text_models_amd" + + +def generate_simple_markdown_report(results: List[BenchmarkResult]) -> str: + """Generate a simplified markdown report without traces and cost columns. + + Skips the first result if it's a warmup run (duplicate batch_size). + """ + model_header = results[0].model_path + if results[0].run_name and results[0].run_name != "default": + model_header += f" ({results[0].run_name})" + + gpu_config = os.getenv("GPU_CONFIG", "AMD") + if gpu_config: + model_header += f" [{gpu_config}]" + + summary = f"### {model_header}\n" + summary += "| batch size | input len | latency (s) | input throughput (tok/s) | output throughput (tok/s) | ITL (ms) |\n" + summary += "| ---------- | --------- | ----------- | ------------------------ | ------------------------- | -------- |\n" + + # Skip first result if it's a warmup (same batch_size as second result) + report_results = ( + results[1:] + if len(results) > 1 and results[0].batch_size == results[1].batch_size + else results + ) + + for result in report_results: + itl = 1 / (result.output_throughput / result.batch_size) * 1000 + summary += f"| {result.batch_size} | {result.input_len} | {result.latency:.2f} | {result.input_throughput:.2f} | {result.output_throughput:.2f} | {itl:.2f} |\n" + + return summary + + +class TestNightlyTextModelsPerfAMD(unittest.TestCase): + """AMD Nightly performance benchmark for text models (2-GPU).""" + + @classmethod + def setUpClass(cls): + cls.models = [] + # Llama-3.1-8B on TP=1 + for model_path in parse_models("meta-llama/Llama-3.1-8B-Instruct"): + cls.models.append( + ModelLaunchSettings( + model_path, + tp_size=1, + extra_args=["--attention-backend", "aiter"], + ) + ) + # Qwen2-57B MoE on TP=2 + for model_path in parse_models("Qwen/Qwen2-57B-A14B-Instruct"): + cls.models.append( + ModelLaunchSettings( + model_path, + tp_size=2, + extra_args=["--attention-backend", "aiter"], + ) + ) + + cls.base_url = DEFAULT_URL_FOR_TEST + # First batch_size=1 is warmup (standalone job, no accuracy test to warm up) + cls.batch_sizes = [1, 1, 8, 16, 64] + cls.input_lens = tuple(_parse_int_list_env("NIGHTLY_INPUT_LENS", "4096")) + cls.output_lens = tuple(_parse_int_list_env("NIGHTLY_OUTPUT_LENS", "512")) + cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url) + cls.runner.setup_profile_directory() + cls.runner.full_report = f"## {cls.__name__}\n" + + def test_bench_one_batch(self): + """Run benchmark for all configured text models.""" + all_model_succeed = True + + try: + for model_setup in self.models: + with self.subTest(model=model_setup.model_path): + other_args = list(model_setup.extra_args or []) + if model_setup.tp_size and model_setup.tp_size > 1: + other_args.extend(["--tp", str(model_setup.tp_size)]) + + result_tuple = self.runner.run_benchmark_for_model( + model_path=model_setup.model_path, + batch_sizes=self.batch_sizes, + input_lens=self.input_lens, + output_lens=self.output_lens, + other_args=other_args, + ) + results = result_tuple[0] + success = result_tuple[1] + + if not success: + all_model_succeed = False + + if results: + self.runner.full_report += ( + generate_simple_markdown_report(results) + "\n" + ) + finally: + self.runner.write_final_report() + + if not all_model_succeed: + raise AssertionError("Some models failed the perf tests.") + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/amd/perf/test_vlms_perf_amd.py b/test/registered/amd/perf/test_vlms_perf_amd.py new file mode 100644 index 000000000..92f6a1fc5 --- /dev/null +++ b/test/registered/amd/perf/test_vlms_perf_amd.py @@ -0,0 +1,145 @@ +"""AMD Nightly performance benchmark for VLM models (2-GPU). + +This test benchmarks Vision-Language Models on AMD MI30x/MI35x with 2 GPUs. + +Registry: nightly-amd-perf-vlm-2-gpu suite + +Example usage: + python -m pytest test_vlms_perf_amd.py -v +""" + +import os +import unittest +import warnings +from typing import List + +from sglang.test.ci.ci_register import register_amd_ci +from sglang.test.nightly_bench_utils import BenchmarkResult +from sglang.test.nightly_utils import NightlyBenchmarkRunner +from sglang.test.test_utils import ( + DEFAULT_URL_FOR_TEST, + ModelLaunchSettings, + _parse_int_list_env, + parse_models, +) + +# Register for AMD CI - VLM models benchmark (~120 min) +register_amd_ci(est_time=7200, suite="nightly-amd-perf-vlm-2-gpu", nightly=True) + +PROFILE_DIR = "performance_profiles_vlms_amd" + +# VLM models suitable for AMD +MODEL_DEFAULTS = [ + ModelLaunchSettings( + "Qwen/Qwen2.5-VL-7B-Instruct", + extra_args=["--mem-fraction-static=0.7"], + ), + ModelLaunchSettings( + "Qwen/Qwen3-VL-30B-A3B-Instruct", + tp_size=2, + ), +] + + +def generate_simple_markdown_report(results: List[BenchmarkResult]) -> str: + """Generate a simplified markdown report without traces and cost columns. + + Skips the first result if it's a warmup run (duplicate batch_size). + """ + model_header = results[0].model_path + if results[0].run_name and results[0].run_name != "default": + model_header += f" ({results[0].run_name})" + + gpu_config = os.getenv("GPU_CONFIG", "AMD") + if gpu_config: + model_header += f" [{gpu_config}]" + + summary = f"### {model_header}\n" + summary += "| batch size | input len | latency (s) | input throughput (tok/s) | output throughput (tok/s) | ITL (ms) |\n" + summary += "| ---------- | --------- | ----------- | ------------------------ | ------------------------- | -------- |\n" + + # Skip first result if it's a warmup (same batch_size as second result) + report_results = ( + results[1:] + if len(results) > 1 and results[0].batch_size == results[1].batch_size + else results + ) + + for result in report_results: + itl = 1 / (result.output_throughput / result.batch_size) * 1000 + summary += f"| {result.batch_size} | {result.input_len} | {result.latency:.2f} | {result.input_throughput:.2f} | {result.output_throughput:.2f} | {itl:.2f} |\n" + + return summary + + +class TestNightlyVLMsPerfAMD(unittest.TestCase): + """AMD Nightly performance benchmark for VLM models (2-GPU).""" + + @classmethod + def setUpClass(cls): + warnings.filterwarnings( + "ignore", category=ResourceWarning, message="unclosed.*socket" + ) + + nightly_vlm_models_str = os.environ.get("NIGHTLY_VLM_MODELS") + if nightly_vlm_models_str: + cls.models = [] + model_paths = parse_models(nightly_vlm_models_str) + for model_path in model_paths: + cls.models.append(ModelLaunchSettings(model_path)) + else: + cls.models = MODEL_DEFAULTS + + cls.base_url = DEFAULT_URL_FOR_TEST + # First batch_size=1 is warmup (standalone job, no accuracy test to warm up) + cls.batch_sizes = _parse_int_list_env("NIGHTLY_VLM_BATCH_SIZES", "1,1,2,8,16") + cls.input_lens = tuple(_parse_int_list_env("NIGHTLY_VLM_INPUT_LENS", "4096")) + cls.output_lens = tuple(_parse_int_list_env("NIGHTLY_VLM_OUTPUT_LENS", "512")) + cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url) + cls.runner.setup_profile_directory() + cls.runner.full_report = f"## {cls.__name__}\n" + + def test_bench_one_batch(self): + """Run benchmark for all configured VLM models.""" + all_model_succeed = True + + try: + for model_setup in self.models: + with self.subTest(model=model_setup.model_path): + other_args = list(model_setup.extra_args or []) + if model_setup.tp_size and model_setup.tp_size > 1: + other_args.extend(["--tp", str(model_setup.tp_size)]) + + # VLMs need additional benchmark args for dataset and trust-remote-code + extra_bench_args = [ + "--trust-remote-code", + "--dataset-name=mmmu", + ] + + result_tuple = self.runner.run_benchmark_for_model( + model_path=model_setup.model_path, + batch_sizes=self.batch_sizes, + input_lens=self.input_lens, + output_lens=self.output_lens, + other_args=other_args, + extra_bench_args=extra_bench_args, + ) + results = result_tuple[0] + success = result_tuple[1] + + if not success: + all_model_succeed = False + + if results: + self.runner.full_report += ( + generate_simple_markdown_report(results) + "\n" + ) + finally: + self.runner.write_final_report() + + if not all_model_succeed: + raise AssertionError("Some models failed the perf tests.") + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/amd/test_deepseek_v32_basic.py b/test/registered/amd/test_deepseek_v32_basic.py index 48bc52615..107947c92 100644 --- a/test/registered/amd/test_deepseek_v32_basic.py +++ b/test/registered/amd/test_deepseek_v32_basic.py @@ -16,7 +16,7 @@ from sglang.test.test_utils import ( ) register_amd_ci(est_time=3600, suite="stage-c-test-large-8-gpu-amd-mi35x") -DEEPSEEK_V32_MODEL_PATH = "deepseek-ai/DeepSeek-V3.2-Exp" +DEEPSEEK_V32_MODEL_PATH = "deepseek-ai/DeepSeek-V3.2" class TestDeepseekV32DP(CustomTestCase): diff --git a/test/registered/amd/test_deepseek_v32_mtp.py b/test/registered/amd/test_deepseek_v32_mtp.py index 04c0c7be2..d9878c016 100644 --- a/test/registered/amd/test_deepseek_v32_mtp.py +++ b/test/registered/amd/test_deepseek_v32_mtp.py @@ -18,7 +18,7 @@ from sglang.test.test_utils import ( ) register_amd_ci(est_time=3600, suite="stage-c-test-large-8-gpu-amd-mi35x") -FULL_DEEPSEEK_V32_MODEL_PATH = "deepseek-ai/DeepSeek-V3.2-Exp" +FULL_DEEPSEEK_V32_MODEL_PATH = "deepseek-ai/DeepSeek-V3.2" class TestDeepseekV32DPMTP(CustomTestCase):