From f8636fbb253a83d268ffa1636eac4e111966376c Mon Sep 17 00:00:00 2001 From: Michael <13900043+michaelzhang-ai@users.noreply.github.com> Date: Wed, 28 Jan 2026 00:55:46 -0800 Subject: [PATCH] [AMD] Add Kimi-K2, DeepSeek-V3.2 tests to nightly CI (#17523) Co-authored-by: YC Tseng --- .github/workflows/nightly-test-amd.yml | 193 +++++++++++--- .../mi35x/test_deepseek_r1_eval_mi35x.py | 3 + .../test_deepseek_r1_mxfp4_eval_mi35x.py | 3 + .../mi35x/test_deepseek_v32_dp_eval_mi35x.py | 119 +++++++++ .../mi35x/test_deepseek_v32_eval_mi35x.py | 3 + .../mi35x/test_deepseek_v32_mtp_eval_mi35x.py | 142 ++++++++++ .../accuracy/mi35x/test_gpt_oss_eval_mi35x.py | 3 + .../mi35x/test_grok1_int4_eval_mi35x.py | 1 + .../accuracy/mi35x/test_grok2_eval_mi35x.py | 1 + .../amd/accuracy/test_deepseek_r1_eval_amd.py | 3 + .../accuracy/test_deepseek_v31_eval_amd.py | 1 + .../accuracy/test_deepseek_v32_dp_eval_amd.py | 122 +++++++++ .../accuracy/test_deepseek_v32_eval_amd.py | 248 ++++++++++++++++++ .../test_deepseek_v32_mtp_eval_amd.py | 142 ++++++++++ .../accuracy/test_deepseek_v32_tc_eval_amd.py | 123 +++++++++ .../amd/accuracy/test_gpt_oss_eval_amd.py | 5 +- .../amd/accuracy/test_grok1_fp8_eval_amd.py | 1 + .../amd/accuracy/test_grok1_int4_eval_amd.py | 1 + .../amd/accuracy/test_grok2_eval_amd.py | 1 + .../amd/accuracy/test_grok_eval_amd.py | 3 + .../amd/accuracy/test_gsm8k_eval_amd.py | 6 +- .../amd/accuracy/test_kimi_k2_eval_amd.py | 101 +++++++ .../amd/accuracy/test_vlms_mmmu_eval_amd.py | 3 +- .../test_deepseek_v32_basic_perf_mi35x.py | 2 + .../mi35x/test_deepseek_v32_mtp_perf_mi35x.py | 62 ++++- .../perf/test_deepseek_v32_basic_perf_amd.py | 142 ++++++++++ .../perf/test_deepseek_v32_mtp_perf_amd.py | 149 +++++++++++ 27 files changed, 1540 insertions(+), 43 deletions(-) create mode 100644 test/registered/amd/accuracy/mi35x/test_deepseek_v32_dp_eval_mi35x.py create mode 100644 test/registered/amd/accuracy/mi35x/test_deepseek_v32_mtp_eval_mi35x.py create mode 100644 test/registered/amd/accuracy/test_deepseek_v32_dp_eval_amd.py create mode 100644 test/registered/amd/accuracy/test_deepseek_v32_eval_amd.py create mode 100644 test/registered/amd/accuracy/test_deepseek_v32_mtp_eval_amd.py create mode 100644 test/registered/amd/accuracy/test_deepseek_v32_tc_eval_amd.py create mode 100644 test/registered/amd/accuracy/test_kimi_k2_eval_amd.py create mode 100644 test/registered/amd/perf/test_deepseek_v32_basic_perf_amd.py create mode 100644 test/registered/amd/perf/test_deepseek_v32_mtp_perf_amd.py diff --git a/.github/workflows/nightly-test-amd.yml b/.github/workflows/nightly-test-amd.yml index 1e0a9bf11..1adb2ab3f 100644 --- a/.github/workflows/nightly-test-amd.yml +++ b/.github/workflows/nightly-test-amd.yml @@ -25,11 +25,13 @@ on: - 'nightly-perf-2-gpu-text' - 'nightly-perf-2-gpu-vlm' - 'nightly-accuracy-8-gpu' - - 'nightly-accuracy-8-gpu-deepseek-r1' # MI30x Accuracy + Performance Tests (combined) - 'nightly-8-gpu-grok1-int4' - 'nightly-8-gpu-grok2' - 'nightly-8-gpu-deepseek-v31' + - 'nightly-8-gpu-deepseek-v32' + - 'nightly-8-gpu-deepseek-v32-mtp' + - 'nightly-8-gpu-kimi-k2' # MI35x jobs - 'nightly-test-1-gpu-mi35x' - 'nightly-accuracy-8-gpu-mi35x' @@ -37,6 +39,7 @@ on: - 'nightly-8-gpu-mi35x-grok2' - 'nightly-8-gpu-mi35x-deepseek-r1-mxfp4' - 'nightly-accuracy-8-gpu-mi35x-deepseek-v32' + - 'nightly-accuracy-8-gpu-mi35x-deepseek-v32-mtp' - 'nightly-perf-8-gpu-mi35x-deepseek-v32-basic' - 'nightly-perf-8-gpu-mi35x-deepseek-v32-mtp' workflow_call: @@ -248,35 +251,6 @@ jobs: echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} - # 8-GPU DeepSeek-R1 Accuracy Test (separate job due to long loading time) - nightly-accuracy-8-gpu-deepseek-r1: - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-accuracy-8-gpu-deepseek-r1') - runs-on: linux-mi325-gpu-8 - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref || github.ref }} - - - name: Setup docker - run: | - touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - - - name: Install dependencies - run: bash scripts/ci/amd/amd_ci_install_dependency.sh - - - name: Accuracy Test (8-GPU DeepSeek-R1) - timeout-minutes: 240 - run: | - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-deepseek-r1 --nightly --timeout-per-file 7200 || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - # ============================================== MI30x Combined Accuracy + Performance Tests ============================================== # 8-GPU Grok1-INT4 (Accuracy + Performance combined) nightly-8-gpu-grok1-int4: @@ -407,6 +381,118 @@ jobs: echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} + # 8-GPU DeepSeek-V3.2 (Basic Accuracy + Perf) + nightly-8-gpu-deepseek-v32: + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-8-gpu-deepseek-v32') + runs-on: linux-mi325-gpu-8 + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref || github.ref }} + + - name: Setup docker + run: | + touch github_summary.md + bash scripts/ci/amd/amd_ci_start_container.sh + env: + GITHUB_WORKSPACE: ${{ github.workspace }} + + - name: Install dependencies + run: bash scripts/ci/amd/amd_ci_install_dependency.sh + + - name: Accuracy Test (8-GPU DeepSeek-V3.2 Basic) + timeout-minutes: 120 + run: | + > github_summary.md # Clear summary file + bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-deepseek-v32 --nightly --timeout-per-file 3600 || TEST_EXIT_CODE=$? + echo "$(> $GITHUB_STEP_SUMMARY || true + exit ${TEST_EXIT_CODE:-0} + + - name: Performance Test (8-GPU DeepSeek-V3.2 Basic) + timeout-minutes: 150 + continue-on-error: true # Perf test failure doesn't fail the job if accuracy passed + run: | + > github_summary.md # Clear summary file + bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-deepseek-v32-basic --nightly --timeout-per-file 5400 || TEST_EXIT_CODE=$? + echo "$(> $GITHUB_STEP_SUMMARY || true + exit ${TEST_EXIT_CODE:-0} + + # 8-GPU DeepSeek-V3.2 MTP (MTP Accuracy + Perf) + nightly-8-gpu-deepseek-v32-mtp: + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-8-gpu-deepseek-v32-mtp') + runs-on: linux-mi325-gpu-8 + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref || github.ref }} + + - name: Setup docker + run: | + touch github_summary.md + bash scripts/ci/amd/amd_ci_start_container.sh + env: + GITHUB_WORKSPACE: ${{ github.workspace }} + + - name: Install dependencies + run: bash scripts/ci/amd/amd_ci_install_dependency.sh + + - name: Accuracy Test (8-GPU DeepSeek-V3.2 MTP) + timeout-minutes: 120 + run: | + > github_summary.md # Clear summary file + bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-deepseek-v32-mtp --nightly --timeout-per-file 3600 || TEST_EXIT_CODE=$? + echo "$(> $GITHUB_STEP_SUMMARY || true + exit ${TEST_EXIT_CODE:-0} + + - name: Performance Test (8-GPU DeepSeek-V3.2 MTP) + timeout-minutes: 180 + continue-on-error: true # Perf test failure doesn't fail the job if accuracy passed + run: | + > github_summary.md # Clear summary file + bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-deepseek-v32-mtp --nightly --timeout-per-file 7200 || TEST_EXIT_CODE=$? + echo "$(> $GITHUB_STEP_SUMMARY || true + exit ${TEST_EXIT_CODE:-0} + + # 8-GPU Kimi-K2 (Accuracy + Speed) + nightly-8-gpu-kimi-k2: + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-8-gpu-kimi-k2') + runs-on: linux-mi325-gpu-8 + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref || github.ref }} + + - name: Setup docker + run: | + touch github_summary.md + bash scripts/ci/amd/amd_ci_start_container.sh + env: + GITHUB_WORKSPACE: ${{ github.workspace }} + + - name: Install dependencies + run: bash scripts/ci/amd/amd_ci_install_dependency.sh + + - name: Accuracy Test (8-GPU Kimi-K2) + timeout-minutes: 120 + run: | + > github_summary.md # Clear summary file + bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-kimi-k2 --nightly --timeout-per-file 3600 || TEST_EXIT_CODE=$? + echo "$(> $GITHUB_STEP_SUMMARY || true + exit ${TEST_EXIT_CODE:-0} + # ============================================== MI35x Tests ============================================== # MI35x 1-GPU tests - platform-agnostic tests that may work on CDNA4 (gfx950) nightly-test-1-gpu-mi35x: @@ -641,6 +727,39 @@ jobs: echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} + # MI35x 8-GPU DeepSeek-V3.2 TP+MTP Accuracy Test + nightly-accuracy-8-gpu-mi35x-deepseek-v32-mtp: + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-accuracy-8-gpu-mi35x-deepseek-v32-mtp') + runs-on: linux-mi35x-gpu-8 + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref || github.ref }} + + - name: Setup docker + run: | + touch github_summary.md + bash scripts/ci/amd/amd_ci_start_container.sh + env: + GITHUB_WORKSPACE: ${{ github.workspace }} + + - name: Install dependencies + run: | + bash scripts/ci/amd/amd_ci_install_dependency.sh + # Install tabulate for run_suite.py (missing in MI35x container) + bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate + + - name: Accuracy Test MI35x (8-GPU DeepSeek-V3.2 TP+MTP) + timeout-minutes: 120 + run: | + > github_summary.md # Clear summary file + bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-mi35x-deepseek-v32-mtp --nightly --timeout-per-file 3600 || TEST_EXIT_CODE=$? + echo "$(> $GITHUB_STEP_SUMMARY || true + exit ${TEST_EXIT_CODE:-0} + # MI35x 8-GPU DeepSeek-V3.2 Performance Test (Basic) nightly-perf-8-gpu-mi35x-deepseek-v32-basic: if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-perf-8-gpu-mi35x-deepseek-v32-basic') @@ -698,12 +817,12 @@ jobs: bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate - name: Performance Test MI35x (8-GPU DeepSeek-V3.2 MTP) - timeout-minutes: 150 + timeout-minutes: 180 run: | > github_summary.md # Clear summary file bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-mi35x-deepseek-v32-mtp --nightly --timeout-per-file 5400 || TEST_EXIT_CODE=$? + python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-mi35x-deepseek-v32-mtp --nightly --timeout-per-file 7200 || TEST_EXIT_CODE=$? echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} @@ -719,11 +838,13 @@ jobs: - nightly-perf-2-gpu-text - nightly-perf-2-gpu-vlm - nightly-accuracy-8-gpu - - nightly-accuracy-8-gpu-deepseek-r1 # MI30x Combined Accuracy + Performance Tests - nightly-8-gpu-grok1-int4 - nightly-8-gpu-grok2 - nightly-8-gpu-deepseek-v31 + - nightly-8-gpu-deepseek-v32 + - nightly-8-gpu-deepseek-v32-mtp + - nightly-8-gpu-kimi-k2 # MI35x jobs - nightly-test-1-gpu-mi35x - nightly-accuracy-8-gpu-mi35x @@ -731,8 +852,10 @@ jobs: - nightly-8-gpu-mi35x-grok2 - nightly-8-gpu-mi35x-deepseek-r1-mxfp4 - nightly-accuracy-8-gpu-mi35x-deepseek-v32 - - nightly-perf-8-gpu-mi35x-deepseek-v32-basic - - nightly-perf-8-gpu-mi35x-deepseek-v32-mtp + - nightly-accuracy-8-gpu-mi35x-deepseek-v32-mtp + # MI35x perf jobs excluded from check - perf failures don't block CI + # - nightly-perf-8-gpu-mi35x-deepseek-v32-basic + # - nightly-perf-8-gpu-mi35x-deepseek-v32-mtp runs-on: ubuntu-latest steps: - name: Check if any job failed diff --git a/test/registered/amd/accuracy/mi35x/test_deepseek_r1_eval_mi35x.py b/test/registered/amd/accuracy/mi35x/test_deepseek_r1_eval_mi35x.py index d4dcc2838..881602036 100644 --- a/test/registered/amd/accuracy/mi35x/test_deepseek_r1_eval_mi35x.py +++ b/test/registered/amd/accuracy/mi35x/test_deepseek_r1_eval_mi35x.py @@ -214,6 +214,9 @@ class TestDeepSeekR1EvalMI35x(unittest.TestCase): ) passed = acc >= config.accuracy_threshold status = "✅ PASS" if passed else "❌ FAIL" + print( + f" accuracy={acc:.3f} threshold={config.accuracy_threshold} {status}" + ) all_results.append( { diff --git a/test/registered/amd/accuracy/mi35x/test_deepseek_r1_mxfp4_eval_mi35x.py b/test/registered/amd/accuracy/mi35x/test_deepseek_r1_mxfp4_eval_mi35x.py index 44491bfb8..d8113a071 100644 --- a/test/registered/amd/accuracy/mi35x/test_deepseek_r1_mxfp4_eval_mi35x.py +++ b/test/registered/amd/accuracy/mi35x/test_deepseek_r1_mxfp4_eval_mi35x.py @@ -239,6 +239,9 @@ class TestDeepSeekR1MXFP4EvalMI35x(unittest.TestCase): ) passed = acc >= config.accuracy_threshold status = "✅ PASS" if passed else "❌ FAIL" + print( + f" accuracy={acc:.3f} threshold={config.accuracy_threshold} {status}" + ) all_results.append( { diff --git a/test/registered/amd/accuracy/mi35x/test_deepseek_v32_dp_eval_mi35x.py b/test/registered/amd/accuracy/mi35x/test_deepseek_v32_dp_eval_mi35x.py new file mode 100644 index 000000000..e196a01c0 --- /dev/null +++ b/test/registered/amd/accuracy/mi35x/test_deepseek_v32_dp_eval_mi35x.py @@ -0,0 +1,119 @@ +"""MI35x DeepSeek-V3.2 DP GSM8K Accuracy Evaluation Test (8-GPU) + +Tests DeepSeek-V3.2 with DP=8 + TP=8 + dp-attention using few-shot +completion benchmark on MI35x. + +Registry: nightly-amd-accuracy-8-gpu-mi35x-deepseek-v32-dp suite +""" + +import os + +# Set HF cache for MI35x +os.environ.setdefault("HF_HOME", "/data2/models/huggingface") +os.environ.setdefault("HF_HUB_CACHE", "/data2/models/huggingface/hub") + +import unittest +from types import SimpleNamespace + +from sglang.srt.utils import kill_process_tree +from sglang.test.ci.ci_register import register_amd_ci +from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.send_one import BenchArgs, send_one_prompt +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + is_in_ci, + popen_launch_server, + write_github_step_summary, +) + +# Register for AMD CI - MI35x DeepSeek-V3.2 DP accuracy test +register_amd_ci( + est_time=3600, + suite="nightly-amd-accuracy-8-gpu-mi35x-deepseek-v32-dp", + nightly=True, +) + +DEEPSEEK_V32_MODEL_PATH = "deepseek-ai/DeepSeek-V3.2" + +# Accuracy threshold +GSM8K_ACCURACY_THRESHOLD = 0.935 + + +class TestDeepseekV32DP(CustomTestCase): + """Test DeepSeek V3.2 with DP=8 + TP=8 + dp-attention. + + This test runs GSM8K evaluation and measures accuracy on MI35x. + """ + + @classmethod + def setUpClass(cls): + cls.model = DEEPSEEK_V32_MODEL_PATH + cls.base_url = DEFAULT_URL_FOR_TEST + other_args = [ + "--trust-remote-code", + "--tp", + "8", + "--dp", + "8", + "--enable-dp-attention", + "--model-loader-extra-config", + '{"enable_multithread_load": true}', + "--nsa-prefill-backend", + "tilelang", + "--nsa-decode-backend", + "tilelang", + ] + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_args, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_a_gsm8k(self): + """GSM8K evaluation for DP configuration. + + Named with 'a' prefix to run first (alphabetically) to warm up the server. + """ + args = SimpleNamespace( + num_shots=20, + data_path=None, + num_questions=1400, + parallel=1400, + max_new_tokens=512, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval_few_shot_gsm8k(args) + print(f"{metrics=}") + + if is_in_ci(): + write_github_step_summary( + f"### test_gsm8k (deepseek-v32 DP MI35x)\n" + f'{metrics["accuracy"]=:.3f}\n' + ) + self.assertGreater(metrics["accuracy"], GSM8K_ACCURACY_THRESHOLD) + + def test_bs_1_speed(self): + """Single batch speed test for DP configuration.""" + args = BenchArgs(port=int(self.base_url.split(":")[-1]), max_new_tokens=2048) + acc_length, speed = send_one_prompt(args) + + print(f"{speed=:.2f}") + + if is_in_ci(): + write_github_step_summary( + f"### test_bs_1_speed (deepseek-v32 DP MI35x)\n" + f"{speed=:.2f} token/s\n" + ) + self.assertGreater(speed, 10) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/amd/accuracy/mi35x/test_deepseek_v32_eval_mi35x.py b/test/registered/amd/accuracy/mi35x/test_deepseek_v32_eval_mi35x.py index 8861355a2..70f851d9d 100644 --- a/test/registered/amd/accuracy/mi35x/test_deepseek_v32_eval_mi35x.py +++ b/test/registered/amd/accuracy/mi35x/test_deepseek_v32_eval_mi35x.py @@ -215,6 +215,9 @@ class TestDeepSeekV32EvalMI35x(unittest.TestCase): ) passed = acc >= config.accuracy_threshold status = "✅ PASS" if passed else "❌ FAIL" + print( + f" accuracy={acc:.3f} threshold={config.accuracy_threshold} {status}" + ) all_results.append( { diff --git a/test/registered/amd/accuracy/mi35x/test_deepseek_v32_mtp_eval_mi35x.py b/test/registered/amd/accuracy/mi35x/test_deepseek_v32_mtp_eval_mi35x.py new file mode 100644 index 000000000..9dd254f84 --- /dev/null +++ b/test/registered/amd/accuracy/mi35x/test_deepseek_v32_mtp_eval_mi35x.py @@ -0,0 +1,142 @@ +"""MI35x DeepSeek-V3.2 TP+MTP GSM8K Accuracy Evaluation Test (8-GPU) + +Tests DeepSeek-V3.2 with TP=8 + MTP (EAGLE speculative decoding) using few-shot +completion benchmark on MI35x. + +Registry: nightly-amd-accuracy-8-gpu-mi35x-deepseek-v32-mtp suite +""" + +import os + +# Set HF cache for MI35x +os.environ.setdefault("HF_HOME", "/data2/models/huggingface") +os.environ.setdefault("HF_HUB_CACHE", "/data2/models/huggingface/hub") + +import unittest +from types import SimpleNamespace + +import requests + +from sglang.srt.utils import kill_process_tree +from sglang.test.ci.ci_register import register_amd_ci +from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.send_one import BenchArgs, send_one_prompt +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + is_in_ci, + popen_launch_server, + write_github_step_summary, +) + +# Register for AMD CI - MI35x DeepSeek-V3.2 TP+MTP accuracy test +register_amd_ci( + est_time=3600, + suite="nightly-amd-accuracy-8-gpu-mi35x-deepseek-v32-mtp", + nightly=True, +) + +DEEPSEEK_V32_MODEL_PATH = "deepseek-ai/DeepSeek-V3.2" + +# Accuracy and performance thresholds +GSM8K_ACCURACY_THRESHOLD = 0.94 +AVG_SPEC_ACCEPT_LENGTH_THRESHOLD = 2.7 + + +class TestDeepseekV32TPMTP(CustomTestCase): + """Test DeepSeek V3.2 with TP=8 + MTP (EAGLE speculative decoding). + + This test runs GSM8K evaluation and measures both accuracy and + speculative decoding acceptance length on MI35x. + """ + + @classmethod + def setUpClass(cls): + cls.model = DEEPSEEK_V32_MODEL_PATH + cls.base_url = DEFAULT_URL_FOR_TEST + other_args = [ + "--trust-remote-code", + "--tp", + "8", + "--speculative-algorithm", + "EAGLE", + "--speculative-num-steps", + "3", + "--speculative-eagle-topk", + "1", + "--speculative-num-draft-tokens", + "4", + "--mem-frac", + "0.7", + "--model-loader-extra-config", + '{"enable_multithread_load": true}', + "--nsa-prefill-backend", + "tilelang", + "--nsa-decode-backend", + "tilelang", + ] + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_args, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_a_gsm8k(self): + """GSM8K evaluation for TP+MTP configuration. + + Named with 'a' prefix to run first (alphabetically) to warm up the server. + """ + requests.get(self.base_url + "/flush_cache") + + args = SimpleNamespace( + num_shots=20, + data_path=None, + num_questions=1400, + parallel=1400, + max_new_tokens=512, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval_few_shot_gsm8k(args) + print(f"{metrics=}") + + server_info = requests.get(self.base_url + "/get_server_info") + avg_spec_accept_length = server_info.json()["internal_states"][0][ + "avg_spec_accept_length" + ] + print(f"{avg_spec_accept_length=}") + + if is_in_ci(): + write_github_step_summary( + f"### test_gsm8k (deepseek-v32 TP+MTP MI35x)\n" + f'{metrics["accuracy"]=:.3f}\n' + f"{avg_spec_accept_length=:.2f}\n" + ) + self.assertGreater(metrics["accuracy"], GSM8K_ACCURACY_THRESHOLD) + self.assertGreater(avg_spec_accept_length, AVG_SPEC_ACCEPT_LENGTH_THRESHOLD) + + def test_bs_1_speed(self): + """Single batch speed test for TP+MTP configuration.""" + args = BenchArgs(port=int(self.base_url.split(":")[-1]), max_new_tokens=2048) + acc_length, speed = send_one_prompt(args) + + print(f"{acc_length=:.2f} {speed=:.2f}") + + if is_in_ci(): + write_github_step_summary( + f"### test_bs_1_speed (deepseek-v32 TP+MTP MI35x)\n" + f"{acc_length=:.2f}\n" + f"{speed=:.2f} token/s\n" + ) + self.assertGreater(acc_length, AVG_SPEC_ACCEPT_LENGTH_THRESHOLD) + self.assertGreater(speed, 55) # Lowered from 60 for AMD MI35x + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/amd/accuracy/mi35x/test_gpt_oss_eval_mi35x.py b/test/registered/amd/accuracy/mi35x/test_gpt_oss_eval_mi35x.py index 4e1451b4e..4c2f8861e 100644 --- a/test/registered/amd/accuracy/mi35x/test_gpt_oss_eval_mi35x.py +++ b/test/registered/amd/accuracy/mi35x/test_gpt_oss_eval_mi35x.py @@ -218,6 +218,9 @@ class TestGptOssEvalMI35x(unittest.TestCase): ) passed = acc >= config.accuracy_threshold status = "✅ PASS" if passed else "❌ FAIL" + print( + f" accuracy={acc:.3f} threshold={config.accuracy_threshold} {status}" + ) all_results.append( { diff --git a/test/registered/amd/accuracy/mi35x/test_grok1_int4_eval_mi35x.py b/test/registered/amd/accuracy/mi35x/test_grok1_int4_eval_mi35x.py index ecee813cc..afe5dd89c 100644 --- a/test/registered/amd/accuracy/mi35x/test_grok1_int4_eval_mi35x.py +++ b/test/registered/amd/accuracy/mi35x/test_grok1_int4_eval_mi35x.py @@ -140,6 +140,7 @@ class TestGrok1INT4EvalMI35x(unittest.TestCase): ) passed = acc >= self.accuracy_threshold status = "✅ PASS" if passed else "❌ FAIL" + print(f" accuracy={acc:.3f} threshold={self.accuracy_threshold} {status}") summary = f"### GROK1-INT4 (MI35x)\n\n" summary += f"| Model | Accuracy | Threshold | Status |\n" diff --git a/test/registered/amd/accuracy/mi35x/test_grok2_eval_mi35x.py b/test/registered/amd/accuracy/mi35x/test_grok2_eval_mi35x.py index 095ba3544..33f826f7c 100644 --- a/test/registered/amd/accuracy/mi35x/test_grok2_eval_mi35x.py +++ b/test/registered/amd/accuracy/mi35x/test_grok2_eval_mi35x.py @@ -142,6 +142,7 @@ class TestGrok2EvalMI35x(unittest.TestCase): ) passed = acc >= self.accuracy_threshold status = "✅ PASS" if passed else "❌ FAIL" + print(f" accuracy={acc:.3f} threshold={self.accuracy_threshold} {status}") summary = f"### GROK2 (MI35x)\n\n" summary += f"| Model | Accuracy | Threshold | Status |\n" diff --git a/test/registered/amd/accuracy/test_deepseek_r1_eval_amd.py b/test/registered/amd/accuracy/test_deepseek_r1_eval_amd.py index 89fff1f74..ee7748669 100644 --- a/test/registered/amd/accuracy/test_deepseek_r1_eval_amd.py +++ b/test/registered/amd/accuracy/test_deepseek_r1_eval_amd.py @@ -273,6 +273,9 @@ class TestDeepSeekR1EvalAMD(unittest.TestCase): ) passed = acc >= config.accuracy_threshold status = "✅ PASS" if passed else "❌ FAIL" + print( + f" accuracy={acc:.3f} threshold={config.accuracy_threshold} {status}" + ) all_results.append( { diff --git a/test/registered/amd/accuracy/test_deepseek_v31_eval_amd.py b/test/registered/amd/accuracy/test_deepseek_v31_eval_amd.py index e50e95426..193ce5b0d 100644 --- a/test/registered/amd/accuracy/test_deepseek_v31_eval_amd.py +++ b/test/registered/amd/accuracy/test_deepseek_v31_eval_amd.py @@ -137,6 +137,7 @@ class TestDeepSeekV31EvalAMD(unittest.TestCase): ) passed = acc >= self.accuracy_threshold status = "✅ PASS" if passed else "❌ FAIL" + print(f" accuracy={acc:.3f} threshold={self.accuracy_threshold} {status}") summary = f"### DeepSeek-V3.1 (MI300X)\n\n" summary += f"| Model | Accuracy | Threshold | Status |\n" diff --git a/test/registered/amd/accuracy/test_deepseek_v32_dp_eval_amd.py b/test/registered/amd/accuracy/test_deepseek_v32_dp_eval_amd.py new file mode 100644 index 000000000..44004abdf --- /dev/null +++ b/test/registered/amd/accuracy/test_deepseek_v32_dp_eval_amd.py @@ -0,0 +1,122 @@ +"""AMD DeepSeek-V3.2 DP GSM8K Accuracy Evaluation Test (8-GPU) + +Tests DeepSeek-V3.2 with DP=8 + TP=8 + dp-attention using few-shot +completion benchmark on MI325/MI300X. + +Registry: nightly-amd-accuracy-8-gpu-deepseek-v32-dp suite +""" + +import os +import unittest +from types import SimpleNamespace + +from sglang.srt.utils import kill_process_tree +from sglang.test.ci.ci_register import register_amd_ci +from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.send_one import BenchArgs, send_one_prompt +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + is_in_ci, + popen_launch_server, + write_github_step_summary, +) + +# Register for AMD CI - DeepSeek-V3.2 DP accuracy test +register_amd_ci( + est_time=5400, + suite="nightly-amd-accuracy-8-gpu-deepseek-v32-dp", + nightly=True, +) + +DEEPSEEK_V32_MODEL_PATH = "deepseek-ai/DeepSeek-V3.2" + +# Accuracy threshold +GSM8K_ACCURACY_THRESHOLD = 0.935 + + +class TestDeepseekV32DP(CustomTestCase): + """Test DeepSeek V3.2 with DP=8 + TP=8 + dp-attention. + + This test runs GSM8K evaluation and measures accuracy on MI325/MI300X. + """ + + @classmethod + def setUpClass(cls): + cls.model = DEEPSEEK_V32_MODEL_PATH + cls.base_url = DEFAULT_URL_FOR_TEST + other_args = [ + "--trust-remote-code", + "--tp", + "8", + "--dp", + "8", + "--enable-dp-attention", + "--attention-backend", + "aiter", + "--chunked-prefill-size", + "131072", + "--mem-fraction-static", + "0.85", + "--model-loader-extra-config", + '{"enable_multithread_load": true}', + "--watchdog-timeout", + "1200", + ] + env = os.environ.copy() + env["SGLANG_USE_AITER"] = "1" + env["SGLANG_USE_ROCM700A"] = "1" + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_args, + env=env, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_a_gsm8k(self): + """GSM8K evaluation for DP configuration. + + Named with 'a' prefix to run first (alphabetically) to warm up the server. + """ + args = SimpleNamespace( + num_shots=20, + data_path=None, + num_questions=1400, + parallel=1400, + max_new_tokens=512, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval_few_shot_gsm8k(args) + print(f"{metrics=}") + + if is_in_ci(): + write_github_step_summary( + f"### test_gsm8k (deepseek-v32 DP MI325)\n" + f'{metrics["accuracy"]=:.3f}\n' + ) + self.assertGreater(metrics["accuracy"], GSM8K_ACCURACY_THRESHOLD) + + def test_bs_1_speed(self): + """Single batch speed test for DP configuration.""" + args = BenchArgs(port=int(self.base_url.split(":")[-1]), max_new_tokens=2048) + acc_length, speed = send_one_prompt(args) + + print(f"{speed=:.2f}") + + if is_in_ci(): + write_github_step_summary( + f"### test_bs_1_speed (deepseek-v32 DP MI325)\n" + f"{speed=:.2f} token/s\n" + ) + self.assertGreater(speed, 10) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/amd/accuracy/test_deepseek_v32_eval_amd.py b/test/registered/amd/accuracy/test_deepseek_v32_eval_amd.py new file mode 100644 index 000000000..1a0521591 --- /dev/null +++ b/test/registered/amd/accuracy/test_deepseek_v32_eval_amd.py @@ -0,0 +1,248 @@ +"""AMD DeepSeek-V3.2 GSM8K Completion Evaluation Test (8-GPU) + +Tests DeepSeek-V3.2 with basic configuration using few-shot completion +benchmark on MI325/MI300X. + +Registry: nightly-amd-accuracy-8-gpu-deepseek-v32 suite +""" + +import ast +import os +import re +import time +import unittest +from dataclasses import dataclass +from typing import List, Optional, Tuple + +import numpy as np + +from sglang.srt.utils import kill_process_tree +from sglang.test.ci.ci_register import register_amd_ci +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + is_in_ci, + popen_launch_server, + write_github_step_summary, +) +from sglang.utils import download_and_cache_file, read_jsonl + +# Register for AMD CI - DeepSeek-V3.2 accuracy test (~60 min for basic only) +register_amd_ci( + est_time=3600, + suite="nightly-amd-accuracy-8-gpu-deepseek-v32", + nightly=True, +) + +INVALID = -9999999 + + +@dataclass +class ModelConfig: + """Configuration for a model to test.""" + + model_path: str + tp_size: int = 8 + accuracy_threshold: float = 0.50 + other_args: Optional[List[str]] = None + env_vars: Optional[dict] = None + timeout: Optional[int] = None + variant: Optional[str] = None + + def __post_init__(self): + if self.other_args is None: + self.other_args = [] + if self.env_vars is None: + self.env_vars = {} + + def get_display_name(self) -> str: + if self.variant: + return f"{self.model_path} ({self.variant})" + return self.model_path + + +# DeepSeek-V3.2 models for MI325/MI300X - basic variant +DEEPSEEK_V32_MODELS = [ + # DeepSeek-V3.2 basic (TP=8 only) + ModelConfig( + model_path="deepseek-ai/DeepSeek-V3.2", + tp_size=8, + accuracy_threshold=0.93, + timeout=3600, + variant="basic", + other_args=[ + "--trust-remote-code", + "--attention-backend", + "aiter", + "--chunked-prefill-size", + "131072", + "--mem-fraction-static", + "0.85", + "--model-loader-extra-config", + '{"enable_multithread_load": true}', + "--watchdog-timeout", + "1200", # 20 minutes for weight loading + ], + env_vars={"SGLANG_USE_AITER": "1"}, + ), +] + + +def get_one_example(lines, i, include_answer): + """Format a single GSM8K example.""" + ret = "Question: " + lines[i]["question"] + "\nAnswer:" + if include_answer: + ret += " " + lines[i]["answer"] + return ret + + +def get_few_shot_examples(lines, k): + """Get k few-shot examples for prompting.""" + ret = "" + for i in range(k): + ret += get_one_example(lines, i, True) + "\n\n" + return ret + + +def get_answer_value(answer_str): + """Extract numerical answer from response.""" + answer_str = answer_str.replace(",", "") + numbers = re.findall(r"\d+", answer_str) + if len(numbers) < 1: + return INVALID + try: + return ast.literal_eval(numbers[-1]) + except SyntaxError: + return INVALID + + +def run_gsm8k_benchmark( + base_url: str, + num_questions: int = 200, + num_shots: int = 5, + parallel: int = 64, +) -> Tuple[float, float, float]: + """Run GSM8K few-shot completion benchmark.""" + import sglang as sgl + from sglang.lang.backend.runtime_endpoint import RuntimeEndpoint + + url = "https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/test.jsonl" + data_path = download_and_cache_file(url) + lines = list(read_jsonl(data_path)) + + few_shot_examples = get_few_shot_examples(lines, num_shots) + + questions = [] + labels = [] + for i in range(len(lines[:num_questions])): + questions.append(get_one_example(lines, i, False)) + labels.append(get_answer_value(lines[i]["answer"])) + assert all(l != INVALID for l in labels) + arguments = [{"question": q} for q in questions] + + @sgl.function + def few_shot_gsm8k(s, question): + s += few_shot_examples + question + s += sgl.gen( + "answer", max_tokens=512, stop=["Question", "Assistant:", "<|separator|>"] + ) + + backend = RuntimeEndpoint(base_url) + sgl.set_default_backend(backend) + + tic = time.perf_counter() + states = few_shot_gsm8k.run_batch( + arguments, temperature=0, num_threads=parallel, progress_bar=True + ) + latency = time.perf_counter() - tic + + preds = [get_answer_value(states[i]["answer"]) for i in range(len(states))] + acc = np.mean(np.array(preds) == np.array(labels)) + invalid = np.mean(np.array(preds) == INVALID) + + return float(acc), float(invalid), float(latency) + + +class TestDeepSeekV32EvalAMD(unittest.TestCase): + """DeepSeek-V3.2 GSM8K Completion Evaluation Test for AMD MI325/MI300X.""" + + @classmethod + def setUpClass(cls): + cls.models = DEEPSEEK_V32_MODELS + cls.base_url = DEFAULT_URL_FOR_TEST + cls.num_questions = int(os.environ.get("GSM8K_NUM_QUESTIONS", "200")) + + def test_deepseek_v32_accuracy(self): + """Test DeepSeek-V3.2 models with GSM8K completion benchmark.""" + all_results = [] + summary = "### DeepSeek-V3.2 Models (MI325)\n\n" + summary += "| Model | Variant | TP | Accuracy | Threshold | Status |\n" + summary += "| ----- | ------- | -- | -------- | --------- | ------ |\n" + + for config in self.models: + display_name = config.get_display_name() + with self.subTest(model=display_name): + print(f"\n{'='*60}") + print(f"Testing: {display_name}") + print(f"{'='*60}") + + env = os.environ.copy() + for key, value in config.env_vars.items(): + env[key] = value + + other_args = list(config.other_args) + other_args.extend(["--tp", str(config.tp_size)]) + timeout = config.timeout or DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH + + try: + process = popen_launch_server( + model=config.model_path, + base_url=self.base_url, + timeout=timeout, + other_args=other_args, + env=env, + ) + + try: + acc, invalid, latency = run_gsm8k_benchmark( + self.base_url, num_questions=self.num_questions + ) + passed = acc >= config.accuracy_threshold + status = "✅ PASS" if passed else "❌ FAIL" + print( + f" accuracy={acc:.3f} threshold={config.accuracy_threshold} {status}" + ) + + all_results.append( + { + "model": display_name, + "accuracy": acc, + "passed": passed, + } + ) + summary += f"| {config.model_path} | {config.variant or 'N/A'} | {config.tp_size} | {acc:.3f} | {config.accuracy_threshold} | {status} |\n" + + finally: + kill_process_tree(process.pid) + + except Exception as e: + summary += f"| {config.model_path} | {config.variant or 'N/A'} | {config.tp_size} | N/A | {config.accuracy_threshold} | ❌ ERROR |\n" + all_results.append( + { + "model": display_name, + "accuracy": None, + "passed": False, + "error": str(e), + } + ) + + if is_in_ci(): + write_github_step_summary(summary) + + failed = [r for r in all_results if not r["passed"]] + if failed: + raise AssertionError(f"Failed models: {[r['model'] for r in failed]}") + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/amd/accuracy/test_deepseek_v32_mtp_eval_amd.py b/test/registered/amd/accuracy/test_deepseek_v32_mtp_eval_amd.py new file mode 100644 index 000000000..6676ab612 --- /dev/null +++ b/test/registered/amd/accuracy/test_deepseek_v32_mtp_eval_amd.py @@ -0,0 +1,142 @@ +"""AMD DeepSeek-V3.2 TP+MTP GSM8K Accuracy Evaluation Test (8-GPU) + +Tests DeepSeek-V3.2 with TP=8 + MTP (EAGLE speculative decoding) using few-shot +completion benchmark on MI325/MI300X. + +Registry: nightly-amd-accuracy-8-gpu-deepseek-v32-mtp suite +""" + +import os +import unittest +from types import SimpleNamespace + +import requests + +from sglang.srt.utils import kill_process_tree +from sglang.test.ci.ci_register import register_amd_ci +from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.send_one import BenchArgs, send_one_prompt +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + is_in_ci, + popen_launch_server, + write_github_step_summary, +) + +# Register for AMD CI - DeepSeek-V3.2 TP+MTP accuracy test +register_amd_ci( + est_time=3600, + suite="nightly-amd-accuracy-8-gpu-deepseek-v32-mtp", + nightly=True, +) + +DEEPSEEK_V32_MODEL_PATH = "deepseek-ai/DeepSeek-V3.2" + +# Accuracy and performance thresholds +GSM8K_ACCURACY_THRESHOLD = 0.94 +AVG_SPEC_ACCEPT_LENGTH_THRESHOLD = 2.7 + + +class TestDeepseekV32TPMTP(CustomTestCase): + """Test DeepSeek V3.2 with TP=8 + MTP (EAGLE speculative decoding). + + This test runs GSM8K evaluation and measures both accuracy and + speculative decoding acceptance length on MI325/MI300X. + """ + + @classmethod + def setUpClass(cls): + cls.model = DEEPSEEK_V32_MODEL_PATH + cls.base_url = DEFAULT_URL_FOR_TEST + other_args = [ + "--trust-remote-code", + "--tp", + "8", + "--attention-backend", + "aiter", + "--chunked-prefill-size", + "131072", + "--speculative-algorithm", + "EAGLE", + "--speculative-num-steps", + "3", + "--speculative-eagle-topk", + "1", + "--speculative-num-draft-tokens", + "4", + "--mem-fraction-static", + "0.7", + "--model-loader-extra-config", + '{"enable_multithread_load": true}', + "--watchdog-timeout", + "1200", + ] + env = os.environ.copy() + env["SGLANG_USE_AITER"] = "1" + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_args, + env=env, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_a_gsm8k(self): + """GSM8K evaluation for TP+MTP configuration. + + Named with 'a' prefix to run first (alphabetically) to warm up the server. + """ + requests.get(self.base_url + "/flush_cache") + + args = SimpleNamespace( + num_shots=20, + data_path=None, + num_questions=1400, + parallel=1400, + max_new_tokens=512, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval_few_shot_gsm8k(args) + print(f"{metrics=}") + + server_info = requests.get(self.base_url + "/get_server_info") + avg_spec_accept_length = server_info.json()["internal_states"][0][ + "avg_spec_accept_length" + ] + print(f"{avg_spec_accept_length=}") + + if is_in_ci(): + write_github_step_summary( + f"### test_gsm8k (deepseek-v32 TP+MTP MI325)\n" + f'{metrics["accuracy"]=:.3f}\n' + f"{avg_spec_accept_length=:.2f}\n" + ) + self.assertGreater(metrics["accuracy"], GSM8K_ACCURACY_THRESHOLD) + self.assertGreater(avg_spec_accept_length, AVG_SPEC_ACCEPT_LENGTH_THRESHOLD) + + def test_bs_1_speed(self): + """Single batch speed test for TP+MTP configuration.""" + args = BenchArgs(port=int(self.base_url.split(":")[-1]), max_new_tokens=2048) + acc_length, speed = send_one_prompt(args) + + print(f"{acc_length=:.2f} {speed=:.2f}") + + if is_in_ci(): + write_github_step_summary( + f"### test_bs_1_speed (deepseek-v32 TP+MTP MI325)\n" + f"{acc_length=:.2f}\n" + f"{speed=:.2f} token/s\n" + ) + self.assertGreater(acc_length, AVG_SPEC_ACCEPT_LENGTH_THRESHOLD) + self.assertGreater(speed, 55) # Lowered from 60 for AMD MI325 + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/amd/accuracy/test_deepseek_v32_tc_eval_amd.py b/test/registered/amd/accuracy/test_deepseek_v32_tc_eval_amd.py new file mode 100644 index 000000000..b1b4df6ee --- /dev/null +++ b/test/registered/amd/accuracy/test_deepseek_v32_tc_eval_amd.py @@ -0,0 +1,123 @@ +"""AMD DeepSeek-V3.2 TC GSM8K Accuracy Evaluation Test (8-GPU) + +Tests DeepSeek-V3.2 with Torch Compile configuration using few-shot +completion benchmark on MI325/MI300X. + +Registry: nightly-amd-accuracy-8-gpu-deepseek-v32-tc suite +""" + +import os +import unittest +from types import SimpleNamespace + +from sglang.srt.utils import kill_process_tree +from sglang.test.ci.ci_register import register_amd_ci +from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.send_one import BenchArgs, send_one_prompt +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + is_in_ci, + popen_launch_server, + write_github_step_summary, +) + +# Register for AMD CI - DeepSeek-V3.2 TC accuracy test +register_amd_ci( + est_time=7200, + suite="nightly-amd-accuracy-8-gpu-deepseek-v32-tc", + nightly=True, +) + +DEEPSEEK_V32_MODEL_PATH = "deepseek-ai/DeepSeek-V3.2" + +# Accuracy threshold +GSM8K_ACCURACY_THRESHOLD = 0.935 + + +class TestDeepseekV32TC(CustomTestCase): + """Test DeepSeek V3.2 with Torch Compile. + + This test runs GSM8K evaluation and measures accuracy on MI325/MI300X. + """ + + @classmethod + def setUpClass(cls): + cls.model = DEEPSEEK_V32_MODEL_PATH + cls.base_url = DEFAULT_URL_FOR_TEST + other_args = [ + "--trust-remote-code", + "--tp", + "8", + "--attention-backend", + "aiter", + "--chunked-prefill-size", + "131072", + "--mem-fraction-static", + "0.70", + "--cuda-graph-max-bs", + "8", + "--enable-torch-compile", + "--disable-cuda-graph", + "--model-loader-extra-config", + '{"enable_multithread_load": true}', + "--watchdog-timeout", + "1200", + ] + env = os.environ.copy() + env["SGLANG_USE_AITER"] = "1" + env["SGLANG_USE_ROCM700A"] = "1" + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_args, + env=env, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_a_gsm8k(self): + """GSM8K evaluation for TC configuration. + + Named with 'a' prefix to run first (alphabetically) to warm up the server. + """ + args = SimpleNamespace( + num_shots=20, + data_path=None, + num_questions=1400, + parallel=1400, + max_new_tokens=512, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval_few_shot_gsm8k(args) + print(f"{metrics=}") + + if is_in_ci(): + write_github_step_summary( + f"### test_gsm8k (deepseek-v32 TC MI325)\n" + f'{metrics["accuracy"]=:.3f}\n' + ) + self.assertGreater(metrics["accuracy"], GSM8K_ACCURACY_THRESHOLD) + + def test_bs_1_speed(self): + """Single batch speed test for TC configuration.""" + args = BenchArgs(port=int(self.base_url.split(":")[-1]), max_new_tokens=2048) + acc_length, speed = send_one_prompt(args) + + print(f"{speed=:.2f}") + + if is_in_ci(): + write_github_step_summary( + f"### test_bs_1_speed (deepseek-v32 TC MI325)\n" + f"{speed=:.2f} token/s\n" + ) + self.assertGreater(speed, 10) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/amd/accuracy/test_gpt_oss_eval_amd.py b/test/registered/amd/accuracy/test_gpt_oss_eval_amd.py index b736361ff..069841256 100644 --- a/test/registered/amd/accuracy/test_gpt_oss_eval_amd.py +++ b/test/registered/amd/accuracy/test_gpt_oss_eval_amd.py @@ -56,7 +56,7 @@ GPT_OSS_MODELS = [ ModelConfig( model_path="lmsys/gpt-oss-20b-bf16", tp_size=8, - accuracy_threshold=0.47, + accuracy_threshold=0.45, other_args=[ "--chunked-prefill-size", "130172", @@ -211,6 +211,9 @@ class TestGptOssEvalAMD(unittest.TestCase): ) passed = acc >= config.accuracy_threshold status = "✅ PASS" if passed else "❌ FAIL" + print( + f" accuracy={acc:.3f} threshold={config.accuracy_threshold} {status}" + ) all_results.append( { diff --git a/test/registered/amd/accuracy/test_grok1_fp8_eval_amd.py b/test/registered/amd/accuracy/test_grok1_fp8_eval_amd.py index 9b9431e6a..5887ee67d 100644 --- a/test/registered/amd/accuracy/test_grok1_fp8_eval_amd.py +++ b/test/registered/amd/accuracy/test_grok1_fp8_eval_amd.py @@ -140,6 +140,7 @@ class TestGrok1FP8EvalAMD(unittest.TestCase): ) passed = acc >= self.accuracy_threshold status = "✅ PASS" if passed else "❌ FAIL" + print(f" accuracy={acc:.3f} threshold={self.accuracy_threshold} {status}") summary = f"### GROK1-FP8 (MI300X)\n\n" summary += f"| Model | Accuracy | Threshold | Status |\n" diff --git a/test/registered/amd/accuracy/test_grok1_int4_eval_amd.py b/test/registered/amd/accuracy/test_grok1_int4_eval_amd.py index 539f4754b..2f0222cc2 100644 --- a/test/registered/amd/accuracy/test_grok1_int4_eval_amd.py +++ b/test/registered/amd/accuracy/test_grok1_int4_eval_amd.py @@ -140,6 +140,7 @@ class TestGrok1INT4EvalAMD(unittest.TestCase): ) passed = acc >= self.accuracy_threshold status = "✅ PASS" if passed else "❌ FAIL" + print(f" accuracy={acc:.3f} threshold={self.accuracy_threshold} {status}") summary = f"### GROK1-INT4 (MI300X)\n\n" summary += f"| Model | Accuracy | Threshold | Status |\n" diff --git a/test/registered/amd/accuracy/test_grok2_eval_amd.py b/test/registered/amd/accuracy/test_grok2_eval_amd.py index 28610c655..192ea78c0 100644 --- a/test/registered/amd/accuracy/test_grok2_eval_amd.py +++ b/test/registered/amd/accuracy/test_grok2_eval_amd.py @@ -140,6 +140,7 @@ class TestGrok2EvalAMD(unittest.TestCase): ) passed = acc >= self.accuracy_threshold status = "✅ PASS" if passed else "❌ FAIL" + print(f" accuracy={acc:.3f} threshold={self.accuracy_threshold} {status}") summary = f"### GROK2 (MI300X)\n\n" summary += f"| Model | Accuracy | Threshold | Status |\n" diff --git a/test/registered/amd/accuracy/test_grok_eval_amd.py b/test/registered/amd/accuracy/test_grok_eval_amd.py index 8ad217349..065cc2c17 100644 --- a/test/registered/amd/accuracy/test_grok_eval_amd.py +++ b/test/registered/amd/accuracy/test_grok_eval_amd.py @@ -251,6 +251,9 @@ class TestGrokEvalAMD(unittest.TestCase): ) passed = acc >= config.accuracy_threshold status = "✅ PASS" if passed else "❌ FAIL" + print( + f" accuracy={acc:.3f} threshold={config.accuracy_threshold} {status}" + ) all_results.append( { diff --git a/test/registered/amd/accuracy/test_gsm8k_eval_amd.py b/test/registered/amd/accuracy/test_gsm8k_eval_amd.py index 5918c6e6e..deb689526 100644 --- a/test/registered/amd/accuracy/test_gsm8k_eval_amd.py +++ b/test/registered/amd/accuracy/test_gsm8k_eval_amd.py @@ -59,9 +59,9 @@ MODEL_SCORE_THRESHOLDS = { "neuralmagic/Meta-Llama-3.1-8B-Instruct-FP8": 0.8, "neuralmagic/Mistral-7B-Instruct-v0.3-FP8": 0.54, "neuralmagic/Meta-Llama-3.1-70B-Instruct-FP8": 0.94, - "neuralmagic/Qwen2-72B-Instruct-FP8": 0.94, - "neuralmagic/Qwen2-57B-A14B-Instruct-FP8": 0.86, - "neuralmagic/Mixtral-8x7B-Instruct-v0.1-FP8": 0.62, + "neuralmagic/Qwen2-72B-Instruct-FP8": 0.92, + "neuralmagic/Qwen2-57B-A14B-Instruct-FP8": 0.81, + "neuralmagic/Mixtral-8x7B-Instruct-v0.1-FP8": 0.57, "neuralmagic/DeepSeek-Coder-V2-Lite-Instruct-FP8": 0.84, } diff --git a/test/registered/amd/accuracy/test_kimi_k2_eval_amd.py b/test/registered/amd/accuracy/test_kimi_k2_eval_amd.py new file mode 100644 index 000000000..cd23d7647 --- /dev/null +++ b/test/registered/amd/accuracy/test_kimi_k2_eval_amd.py @@ -0,0 +1,101 @@ +"""AMD Kimi-K2 GSM8K Completion Evaluation Test (8-GPU) + +Tests moonshotai/Kimi-K2-Instruct-0905 with GSM8K few-shot benchmark on MI325. + +Registry: nightly-amd-accuracy-8-gpu-kimi-k2 suite +""" + +import os +import unittest +from types import SimpleNamespace + +import requests + +from sglang.srt.utils import kill_process_tree +from sglang.test.ci.ci_register import register_amd_ci +from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.test_utils import ( + DEFAULT_URL_FOR_TEST, + CustomTestCase, + is_in_ci, + popen_launch_server, + write_github_step_summary, +) + +# Register for AMD CI - Kimi K2 accuracy test (~60 min) +register_amd_ci(est_time=3600, suite="nightly-amd-accuracy-8-gpu-kimi-k2", nightly=True) + +KIMI_K2_MODEL_PATH = "moonshotai/Kimi-K2-Instruct-0905" +SERVER_LAUNCH_TIMEOUT = 3600 +ACCURACY_THRESHOLD = 0.94 + + +class TestKimiK2EvalAMD(CustomTestCase): + """Kimi-K2 GSM8K Completion Evaluation Test for AMD MI325.""" + + @classmethod + def setUpClass(cls): + cls.model = KIMI_K2_MODEL_PATH + cls.base_url = DEFAULT_URL_FOR_TEST + other_args = [ + "--tp", + "8", + "--decode-attention-backend", + "triton", + "--prefill-attention-backend", + "aiter", + "--trust-remote-code", + "--model-loader-extra-config", + '{"enable_multithread_load": true}', + ] + env = os.environ.copy() + env["SGLANG_USE_AITER"] = "1" + env["SGLANG_ROCM_FUSED_DECODE_MLA"] = "0" + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=SERVER_LAUNCH_TIMEOUT, + other_args=other_args, + env=env, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_kimi_k2_gsm8k_accuracy(self): + """Test Kimi-K2 with GSM8K few-shot completion benchmark.""" + requests.get(self.base_url + "/flush_cache") + + args = SimpleNamespace( + num_shots=8, + data_path=None, + num_questions=1319, + parallel=1319, + max_new_tokens=512, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval_few_shot_gsm8k(args) + acc = metrics["accuracy"] + + passed = acc >= ACCURACY_THRESHOLD + status = "✅ PASS" if passed else "❌ FAIL" + print(f" accuracy={acc:.3f} threshold={ACCURACY_THRESHOLD} {status}") + + if is_in_ci(): + summary = "### Kimi-K2 Model (MI325)\n\n" + summary += "| Model | TP | Accuracy | Threshold | Status |\n" + summary += "| ----- | -- | -------- | --------- | ------ |\n" + summary += f"| {KIMI_K2_MODEL_PATH} | 8 | {acc:.3f} | {ACCURACY_THRESHOLD} | {status} |\n" + write_github_step_summary(summary) + + self.assertGreaterEqual( + acc, + ACCURACY_THRESHOLD, + f"Kimi-K2 accuracy {acc:.3f} below threshold {ACCURACY_THRESHOLD}", + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/amd/accuracy/test_vlms_mmmu_eval_amd.py b/test/registered/amd/accuracy/test_vlms_mmmu_eval_amd.py index 847f11b82..ec1e49476 100644 --- a/test/registered/amd/accuracy/test_vlms_mmmu_eval_amd.py +++ b/test/registered/amd/accuracy/test_vlms_mmmu_eval_amd.py @@ -127,7 +127,8 @@ TRITON_ATTENTION_MODELS = { # Models known to fail on AMD - exclude from testing AMD_FAILING_VLM_MODELS = { - # Add models here as they are discovered to fail + # GLM-4.1V processor not registered yet (Glm4vForConditionalGeneration) + "zai-org/GLM-4.1V-9B-Thinking", } diff --git a/test/registered/amd/perf/mi35x/test_deepseek_v32_basic_perf_mi35x.py b/test/registered/amd/perf/mi35x/test_deepseek_v32_basic_perf_mi35x.py index 96365d9c7..a9bfad3c9 100644 --- a/test/registered/amd/perf/mi35x/test_deepseek_v32_basic_perf_mi35x.py +++ b/test/registered/amd/perf/mi35x/test_deepseek_v32_basic_perf_mi35x.py @@ -93,6 +93,8 @@ class TestNightlyDeepseekV32BasicPerformance(unittest.TestCase): "0.85", "--model-loader-extra-config", '{"enable_multithread_load": true}', + "--watchdog-timeout", + "1200", ], } diff --git a/test/registered/amd/perf/mi35x/test_deepseek_v32_mtp_perf_mi35x.py b/test/registered/amd/perf/mi35x/test_deepseek_v32_mtp_perf_mi35x.py index 01aa19bb7..39ec2cc18 100644 --- a/test/registered/amd/perf/mi35x/test_deepseek_v32_mtp_perf_mi35x.py +++ b/test/registered/amd/perf/mi35x/test_deepseek_v32_mtp_perf_mi35x.py @@ -13,12 +13,17 @@ Example usage: import os import unittest -from typing import List +from typing import List, Optional, Tuple +from sglang.srt.utils import kill_process_tree from sglang.test.ci.ci_register import register_amd_ci from sglang.test.nightly_bench_utils import BenchmarkResult from sglang.test.nightly_utils import NightlyBenchmarkRunner -from sglang.test.test_utils import DEFAULT_URL_FOR_TEST, _parse_int_list_env +from sglang.test.test_utils import ( + DEFAULT_URL_FOR_TEST, + _parse_int_list_env, + popen_launch_server, +) # Register for AMD CI - DeepSeek-V3.2 MTP benchmark (~90 min) register_amd_ci( @@ -57,11 +62,58 @@ def generate_simple_markdown_report(results: List[BenchmarkResult]) -> str: return summary +def _run_benchmark_with_timeout( + runner: NightlyBenchmarkRunner, + model_path: str, + batch_sizes: List[int], + input_lens: Tuple[int, ...], + output_lens: Tuple[int, ...], + other_args: List[str], + variant: str, + extra_bench_args: Optional[List[str]], + timeout: int, +) -> Tuple[List[BenchmarkResult], bool, Optional[float]]: + """Run benchmark with a custom server launch timeout.""" + model_description = f"{model_path}" + (f" ({variant})" if variant else "") + process = popen_launch_server( + model=model_path, + base_url=runner.base_url, + other_args=other_args, + timeout=timeout, + ) + try: + profile_path_prefix, json_output_file = runner.generate_profile_filename( + model_path, variant + ) + bench_args = list(extra_bench_args) if extra_bench_args else [] + if variant: + bench_args.extend(["--run-name", variant]) + command = runner.build_benchmark_command( + model_path, + batch_sizes, + input_lens, + output_lens, + profile_path_prefix, + json_output_file, + extra_args=bench_args, + ) + _, cmd_success = runner.run_benchmark_command(command, model_description) + if not cmd_success: + return [], False, None + benchmark_results, load_success = runner.load_benchmark_results( + json_output_file, model_description + ) + return benchmark_results, load_success, None + finally: + kill_process_tree(process.pid) + + # Model path can be overridden via environment variable DEEPSEEK_V32_MODEL_PATH = os.environ.get( "DEEPSEEK_V32_MODEL_PATH", "deepseek-ai/DeepSeek-V3.2" ) PROFILE_DIR = "performance_profiles_deepseek_v32_mtp" +SERVER_LAUNCH_TIMEOUT = 5400 class TestNightlyDeepseekV32MTPPerformance(unittest.TestCase): @@ -102,6 +154,8 @@ class TestNightlyDeepseekV32MTPPerformance(unittest.TestCase): "0.7", "--model-loader-extra-config", '{"enable_multithread_load": true}', + "--watchdog-timeout", + "1200", ], } @@ -113,7 +167,8 @@ class TestNightlyDeepseekV32MTPPerformance(unittest.TestCase): def test_bench_one_batch(self): """Run benchmark for MTP variant.""" try: - result_tuple = self.runner.run_benchmark_for_model( + result_tuple = _run_benchmark_with_timeout( + runner=self.runner, model_path=self.model, batch_sizes=self.batch_sizes, input_lens=self.input_lens, @@ -121,6 +176,7 @@ class TestNightlyDeepseekV32MTPPerformance(unittest.TestCase): other_args=self.variant_config["other_args"], variant=self.variant_config["name"], extra_bench_args=["--trust-remote-code"], + timeout=SERVER_LAUNCH_TIMEOUT, ) results = result_tuple[0] success = result_tuple[1] diff --git a/test/registered/amd/perf/test_deepseek_v32_basic_perf_amd.py b/test/registered/amd/perf/test_deepseek_v32_basic_perf_amd.py new file mode 100644 index 000000000..56d0fc433 --- /dev/null +++ b/test/registered/amd/perf/test_deepseek_v32_basic_perf_amd.py @@ -0,0 +1,142 @@ +"""AMD Nightly performance benchmark for DeepSeek-V3.2 model (basic variant). + +This test benchmarks the DeepSeek-V3.2 model with basic TP=8 configuration on 8 GPUs. + +The model path can be configured via DEEPSEEK_V32_MODEL_PATH environment variable. + +Registry: nightly-perf-8-gpu-deepseek-v32-basic suite + +Example usage: + DEEPSEEK_V32_MODEL_PATH=deepseek-ai/DeepSeek-V3.2 python -m pytest test_deepseek_v32_basic_perf_amd.py -v +""" + +import os +import unittest +from typing import List + +from sglang.test.ci.ci_register import register_amd_ci +from sglang.test.nightly_bench_utils import BenchmarkResult +from sglang.test.nightly_utils import NightlyBenchmarkRunner +from sglang.test.test_utils import DEFAULT_URL_FOR_TEST, _parse_int_list_env + +# Register for AMD CI - DeepSeek-V3.2 basic benchmark (~90 min) +register_amd_ci( + est_time=5400, suite="nightly-perf-8-gpu-deepseek-v32-basic", nightly=True +) + + +def generate_simple_markdown_report(results: List[BenchmarkResult]) -> str: + """Generate a simplified markdown report without traces and cost columns. + + Skips the first result if it's a warmup run (duplicate batch_size). + """ + model_header = results[0].model_path + if results[0].run_name and results[0].run_name != "default": + model_header += f" ({results[0].run_name})" + + gpu_config = os.getenv("GPU_CONFIG", "MI325") + if gpu_config: + model_header += f" [{gpu_config}]" + + summary = f"### {model_header}\n" + summary += "| batch size | input len | latency (s) | input throughput (tok/s) | output throughput (tok/s) | ITL (ms) |\n" + summary += "| ---------- | --------- | ----------- | ------------------------ | ------------------------- | -------- |\n" + + # Skip first result if it's a warmup (same batch_size as second result) + report_results = ( + results[1:] + if len(results) > 1 and results[0].batch_size == results[1].batch_size + else results + ) + + for result in report_results: + itl = 1 / (result.output_throughput / result.batch_size) * 1000 + summary += f"| {result.batch_size} | {result.input_len} | {result.latency:.2f} | {result.input_throughput:.2f} | {result.output_throughput:.2f} | {itl:.2f} |\n" + + return summary + + +# Model path can be overridden via environment variable +DEEPSEEK_V32_MODEL_PATH = os.environ.get( + "DEEPSEEK_V32_MODEL_PATH", "deepseek-ai/DeepSeek-V3.2" +) +PROFILE_DIR = "performance_profiles_deepseek_v32_basic_mi325" + + +class TestNightlyDeepseekV32BasicPerformance(unittest.TestCase): + """AMD Nightly performance benchmark for DeepSeek-V3.2 model (basic variant). + + Tests the DeepSeek-V3.2 model with basic TP=8 configuration on MI325/MI300X. + """ + + @classmethod + def setUpClass(cls): + cls.model = DEEPSEEK_V32_MODEL_PATH + cls.base_url = DEFAULT_URL_FOR_TEST + cls.batch_sizes = [1, 8, 16, 64] + cls.input_lens = tuple(_parse_int_list_env("NIGHTLY_INPUT_LENS", "4096")) + cls.output_lens = tuple(_parse_int_list_env("NIGHTLY_OUTPUT_LENS", "512")) + + # Basic variant configuration for DeepSeek-V3.2 + # MI325 uses aiter attention backend + cls.variant_config = { + "name": "basic", + "other_args": [ + "--trust-remote-code", + "--tp", + "8", + "--attention-backend", + "aiter", + "--chunked-prefill-size", + "131072", + "--mem-fraction-static", + "0.85", + "--model-loader-extra-config", + '{"enable_multithread_load": true}', + "--watchdog-timeout", + "1200", + ], + "env_vars": {"SGLANG_USE_AITER": "1"}, + } + + cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url) + cls.runner.setup_profile_directory() + # Override full_report to remove traces help text + cls.runner.full_report = f"## {cls.__name__}\n" + + def test_bench_one_batch(self): + """Run benchmark for basic variant.""" + try: + result_tuple = self.runner.run_benchmark_for_model( + model_path=self.model, + batch_sizes=self.batch_sizes, + input_lens=self.input_lens, + output_lens=self.output_lens, + other_args=self.variant_config["other_args"], + variant=self.variant_config["name"], + extra_bench_args=["--trust-remote-code"], + ) + results = result_tuple[0] + success = result_tuple[1] + avg_spec_accept_length = result_tuple[2] if len(result_tuple) > 2 else None + + # Log speculative decoding accept length + if avg_spec_accept_length is not None: + print(f" avg_spec_accept_length={avg_spec_accept_length:.2f}") + + # Use simplified report format without traces + if results: + self.runner.full_report += ( + generate_simple_markdown_report(results) + "\n" + ) + + if not success: + raise AssertionError( + f"Benchmark failed for {self.model} (basic variant)" + ) + finally: + self.runner.write_final_report() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/amd/perf/test_deepseek_v32_mtp_perf_amd.py b/test/registered/amd/perf/test_deepseek_v32_mtp_perf_amd.py new file mode 100644 index 000000000..dc316c77d --- /dev/null +++ b/test/registered/amd/perf/test_deepseek_v32_mtp_perf_amd.py @@ -0,0 +1,149 @@ +"""AMD Nightly performance benchmark for DeepSeek-V3.2 model (MTP variant). + +This test benchmarks the DeepSeek-V3.2 model with MTP (EAGLE speculative decoding) +configuration on 8 GPUs. + +The model path can be configured via DEEPSEEK_V32_MODEL_PATH environment variable. + +Registry: nightly-perf-8-gpu-deepseek-v32-mtp suite + +Example usage: + DEEPSEEK_V32_MODEL_PATH=deepseek-ai/DeepSeek-V3.2 python -m pytest test_deepseek_v32_mtp_perf_amd.py -v +""" + +import os +import unittest +from typing import List + +from sglang.test.ci.ci_register import register_amd_ci +from sglang.test.nightly_bench_utils import BenchmarkResult +from sglang.test.nightly_utils import NightlyBenchmarkRunner +from sglang.test.test_utils import DEFAULT_URL_FOR_TEST, _parse_int_list_env + +# Register for AMD CI - DeepSeek-V3.2 MTP benchmark (~120 min) +register_amd_ci( + est_time=7200, suite="nightly-perf-8-gpu-deepseek-v32-mtp", nightly=True +) + + +def generate_simple_markdown_report(results: List[BenchmarkResult]) -> str: + """Generate a simplified markdown report without traces and cost columns. + + Skips the first result if it's a warmup run (duplicate batch_size). + """ + model_header = results[0].model_path + if results[0].run_name and results[0].run_name != "default": + model_header += f" ({results[0].run_name})" + + gpu_config = os.getenv("GPU_CONFIG", "MI325") + if gpu_config: + model_header += f" [{gpu_config}]" + + summary = f"### {model_header}\n" + summary += "| batch size | input len | latency (s) | input throughput (tok/s) | output throughput (tok/s) | ITL (ms) |\n" + summary += "| ---------- | --------- | ----------- | ------------------------ | ------------------------- | -------- |\n" + + # Skip first result if it's a warmup (same batch_size as second result) + report_results = ( + results[1:] + if len(results) > 1 and results[0].batch_size == results[1].batch_size + else results + ) + + for result in report_results: + itl = 1 / (result.output_throughput / result.batch_size) * 1000 + summary += f"| {result.batch_size} | {result.input_len} | {result.latency:.2f} | {result.input_throughput:.2f} | {result.output_throughput:.2f} | {itl:.2f} |\n" + + return summary + + +# Model path can be overridden via environment variable +DEEPSEEK_V32_MODEL_PATH = os.environ.get( + "DEEPSEEK_V32_MODEL_PATH", "deepseek-ai/DeepSeek-V3.2" +) +PROFILE_DIR = "performance_profiles_deepseek_v32_mtp_mi325" + + +class TestNightlyDeepseekV32MTPPerformance(unittest.TestCase): + """AMD Nightly performance benchmark for DeepSeek-V3.2 model (MTP variant). + + Tests the DeepSeek-V3.2 model with MTP (EAGLE speculative decoding) on MI325/MI300X. + """ + + @classmethod + def setUpClass(cls): + cls.model = DEEPSEEK_V32_MODEL_PATH + cls.base_url = DEFAULT_URL_FOR_TEST + cls.batch_sizes = [1, 8, 16, 64] + cls.input_lens = tuple(_parse_int_list_env("NIGHTLY_INPUT_LENS", "4096")) + cls.output_lens = tuple(_parse_int_list_env("NIGHTLY_OUTPUT_LENS", "512")) + + # MTP variant configuration for DeepSeek-V3.2 + # MI325 uses aiter attention backend + EAGLE speculative decoding + cls.variant_config = { + "name": "mtp", + "other_args": [ + "--trust-remote-code", + "--tp", + "8", + "--attention-backend", + "aiter", + "--chunked-prefill-size", + "131072", + "--speculative-algorithm", + "EAGLE", + "--speculative-num-steps", + "3", + "--speculative-eagle-topk", + "1", + "--speculative-num-draft-tokens", + "4", + "--mem-fraction-static", + "0.7", + "--model-loader-extra-config", + '{"enable_multithread_load": true}', + "--watchdog-timeout", + "1200", + ], + "env_vars": {"SGLANG_USE_AITER": "1"}, + } + + cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url) + cls.runner.setup_profile_directory() + # Override full_report to remove traces help text + cls.runner.full_report = f"## {cls.__name__}\n" + + def test_bench_one_batch(self): + """Run benchmark for MTP variant.""" + try: + result_tuple = self.runner.run_benchmark_for_model( + model_path=self.model, + batch_sizes=self.batch_sizes, + input_lens=self.input_lens, + output_lens=self.output_lens, + other_args=self.variant_config["other_args"], + variant=self.variant_config["name"], + extra_bench_args=["--trust-remote-code"], + ) + results = result_tuple[0] + success = result_tuple[1] + avg_spec_accept_length = result_tuple[2] if len(result_tuple) > 2 else None + + # Log speculative decoding accept length + if avg_spec_accept_length is not None: + print(f" avg_spec_accept_length={avg_spec_accept_length:.2f}") + + # Use simplified report format without traces + if results: + self.runner.full_report += ( + generate_simple_markdown_report(results) + "\n" + ) + + if not success: + raise AssertionError(f"Benchmark failed for {self.model} (MTP variant)") + finally: + self.runner.write_final_report() + + +if __name__ == "__main__": + unittest.main()