chenchenghao/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[AMD] Add Kimi-K2, DeepSeek-V3.2 tests to nightly CI (#17523)

Browse Source 
Co-authored-by: YC Tseng <yctseng@amd.com>
This commit is contained in:
Michael
2026-01-28 00:55:46 -08:00
committed by GitHub
parent 6077de1237
commit f8636fbb25
27 changed files with 1540 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files

193
.github/workflows/nightly-test-amd.yml vendored
View File

@@ -25,11 +25,13 @@ on:
- 'nightly-perf-2-gpu-text'
- 'nightly-perf-2-gpu-vlm'
- 'nightly-accuracy-8-gpu'
- 'nightly-accuracy-8-gpu-deepseek-r1'
# MI30x Accuracy + Performance Tests (combined)
- 'nightly-8-gpu-grok1-int4'
- 'nightly-8-gpu-grok2'
- 'nightly-8-gpu-deepseek-v31'
- 'nightly-8-gpu-deepseek-v32'
- 'nightly-8-gpu-deepseek-v32-mtp'
- 'nightly-8-gpu-kimi-k2'
# MI35x jobs
- 'nightly-test-1-gpu-mi35x'
- 'nightly-accuracy-8-gpu-mi35x'
@@ -37,6 +39,7 @@ on:
- 'nightly-8-gpu-mi35x-grok2'
- 'nightly-8-gpu-mi35x-deepseek-r1-mxfp4'
- 'nightly-accuracy-8-gpu-mi35x-deepseek-v32'
- 'nightly-accuracy-8-gpu-mi35x-deepseek-v32-mtp'
- 'nightly-perf-8-gpu-mi35x-deepseek-v32-basic'
- 'nightly-perf-8-gpu-mi35x-deepseek-v32-mtp'
workflow_call:
@@ -248,35 +251,6 @@ jobs:
echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
exit ${TEST_EXIT_CODE:-0}
# 8-GPU DeepSeek-R1 Accuracy Test (separate job due to long loading time)
nightly-accuracy-8-gpu-deepseek-r1:
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-accuracy-8-gpu-deepseek-r1')
runs-on: linux-mi325-gpu-8
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
ref: ${{ inputs.ref || github.ref }}
- name: Setup docker
run: |
touch github_summary.md
bash scripts/ci/amd/amd_ci_start_container.sh
env:
GITHUB_WORKSPACE: ${{ github.workspace }}
- name: Install dependencies
run: bash scripts/ci/amd/amd_ci_install_dependency.sh
- name: Accuracy Test (8-GPU DeepSeek-R1)
timeout-minutes: 240
run: |
bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
-e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-deepseek-r1 --nightly --timeout-per-file 7200 || TEST_EXIT_CODE=$?
echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
exit ${TEST_EXIT_CODE:-0}
# ============================================== MI30x Combined Accuracy + Performance Tests ==============================================
# 8-GPU Grok1-INT4 (Accuracy + Performance combined)
nightly-8-gpu-grok1-int4:
@@ -407,6 +381,118 @@ jobs:
echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
exit ${TEST_EXIT_CODE:-0}
# 8-GPU DeepSeek-V3.2 (Basic Accuracy + Perf)
nightly-8-gpu-deepseek-v32:
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-8-gpu-deepseek-v32')
runs-on: linux-mi325-gpu-8
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
ref: ${{ inputs.ref || github.ref }}
- name: Setup docker
run: |
touch github_summary.md
bash scripts/ci/amd/amd_ci_start_container.sh
env:
GITHUB_WORKSPACE: ${{ github.workspace }}
- name: Install dependencies
run: bash scripts/ci/amd/amd_ci_install_dependency.sh
- name: Accuracy Test (8-GPU DeepSeek-V3.2 Basic)
timeout-minutes: 120
run: |
> github_summary.md # Clear summary file
bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
-e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-deepseek-v32 --nightly --timeout-per-file 3600 || TEST_EXIT_CODE=$?
echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
exit ${TEST_EXIT_CODE:-0}
- name: Performance Test (8-GPU DeepSeek-V3.2 Basic)
timeout-minutes: 150
continue-on-error: true # Perf test failure doesn't fail the job if accuracy passed
run: |
> github_summary.md # Clear summary file
bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
-e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-deepseek-v32-basic --nightly --timeout-per-file 5400 || TEST_EXIT_CODE=$?
echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
exit ${TEST_EXIT_CODE:-0}
# 8-GPU DeepSeek-V3.2 MTP (MTP Accuracy + Perf)
nightly-8-gpu-deepseek-v32-mtp:
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-8-gpu-deepseek-v32-mtp')
runs-on: linux-mi325-gpu-8
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
ref: ${{ inputs.ref || github.ref }}
- name: Setup docker
run: |
touch github_summary.md
bash scripts/ci/amd/amd_ci_start_container.sh
env:
GITHUB_WORKSPACE: ${{ github.workspace }}
- name: Install dependencies
run: bash scripts/ci/amd/amd_ci_install_dependency.sh
- name: Accuracy Test (8-GPU DeepSeek-V3.2 MTP)
timeout-minutes: 120
run: |
> github_summary.md # Clear summary file
bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
-e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-deepseek-v32-mtp --nightly --timeout-per-file 3600 || TEST_EXIT_CODE=$?
echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
exit ${TEST_EXIT_CODE:-0}
- name: Performance Test (8-GPU DeepSeek-V3.2 MTP)
timeout-minutes: 180
continue-on-error: true # Perf test failure doesn't fail the job if accuracy passed
run: |
> github_summary.md # Clear summary file
bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
-e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-deepseek-v32-mtp --nightly --timeout-per-file 7200 || TEST_EXIT_CODE=$?
echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
exit ${TEST_EXIT_CODE:-0}
# 8-GPU Kimi-K2 (Accuracy + Speed)
nightly-8-gpu-kimi-k2:
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-8-gpu-kimi-k2')
runs-on: linux-mi325-gpu-8
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
ref: ${{ inputs.ref || github.ref }}
- name: Setup docker
run: |
touch github_summary.md
bash scripts/ci/amd/amd_ci_start_container.sh
env:
GITHUB_WORKSPACE: ${{ github.workspace }}
- name: Install dependencies
run: bash scripts/ci/amd/amd_ci_install_dependency.sh
- name: Accuracy Test (8-GPU Kimi-K2)
timeout-minutes: 120
run: |
> github_summary.md # Clear summary file
bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
-e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-kimi-k2 --nightly --timeout-per-file 3600 || TEST_EXIT_CODE=$?
echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
exit ${TEST_EXIT_CODE:-0}
# ============================================== MI35x Tests ==============================================
# MI35x 1-GPU tests - platform-agnostic tests that may work on CDNA4 (gfx950)
nightly-test-1-gpu-mi35x:
@@ -641,6 +727,39 @@ jobs:
echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
exit ${TEST_EXIT_CODE:-0}
# MI35x 8-GPU DeepSeek-V3.2 TP+MTP Accuracy Test
nightly-accuracy-8-gpu-mi35x-deepseek-v32-mtp:
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-accuracy-8-gpu-mi35x-deepseek-v32-mtp')
runs-on: linux-mi35x-gpu-8
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
ref: ${{ inputs.ref || github.ref }}
- name: Setup docker
run: |
touch github_summary.md
bash scripts/ci/amd/amd_ci_start_container.sh
env:
GITHUB_WORKSPACE: ${{ github.workspace }}
- name: Install dependencies
run: |
bash scripts/ci/amd/amd_ci_install_dependency.sh
# Install tabulate for run_suite.py (missing in MI35x container)
bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate
- name: Accuracy Test MI35x (8-GPU DeepSeek-V3.2 TP+MTP)
timeout-minutes: 120
run: |
> github_summary.md # Clear summary file
bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
-e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-mi35x-deepseek-v32-mtp --nightly --timeout-per-file 3600 || TEST_EXIT_CODE=$?
echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
exit ${TEST_EXIT_CODE:-0}
# MI35x 8-GPU DeepSeek-V3.2 Performance Test (Basic)
nightly-perf-8-gpu-mi35x-deepseek-v32-basic:
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-perf-8-gpu-mi35x-deepseek-v32-basic')
@@ -698,12 +817,12 @@ jobs:
bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate
- name: Performance Test MI35x (8-GPU DeepSeek-V3.2 MTP)
timeout-minutes: 150
timeout-minutes: 180
run: |
> github_summary.md # Clear summary file
bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
-e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-mi35x-deepseek-v32-mtp --nightly --timeout-per-file 5400 || TEST_EXIT_CODE=$?
python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-mi35x-deepseek-v32-mtp --nightly --timeout-per-file 7200 || TEST_EXIT_CODE=$?
echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
exit ${TEST_EXIT_CODE:-0}
@@ -719,11 +838,13 @@ jobs:
- nightly-perf-2-gpu-text
- nightly-perf-2-gpu-vlm
- nightly-accuracy-8-gpu
- nightly-accuracy-8-gpu-deepseek-r1
# MI30x Combined Accuracy + Performance Tests
- nightly-8-gpu-grok1-int4
- nightly-8-gpu-grok2
- nightly-8-gpu-deepseek-v31
- nightly-8-gpu-deepseek-v32
- nightly-8-gpu-deepseek-v32-mtp
- nightly-8-gpu-kimi-k2
# MI35x jobs
- nightly-test-1-gpu-mi35x
- nightly-accuracy-8-gpu-mi35x
@@ -731,8 +852,10 @@ jobs:
- nightly-8-gpu-mi35x-grok2
- nightly-8-gpu-mi35x-deepseek-r1-mxfp4
- nightly-accuracy-8-gpu-mi35x-deepseek-v32
- nightly-perf-8-gpu-mi35x-deepseek-v32-basic
- nightly-perf-8-gpu-mi35x-deepseek-v32-mtp
- nightly-accuracy-8-gpu-mi35x-deepseek-v32-mtp
# MI35x perf jobs excluded from check - perf failures don't block CI
# - nightly-perf-8-gpu-mi35x-deepseek-v32-basic
# - nightly-perf-8-gpu-mi35x-deepseek-v32-mtp
runs-on: ubuntu-latest
steps:
- name: Check if any job failed

3
test/registered/amd/accuracy/mi35x/test_deepseek_r1_eval_mi35x.py
View File

@@ -214,6 +214,9 @@ class TestDeepSeekR1EvalMI35x(unittest.TestCase):
)
passed = acc >= config.accuracy_threshold
status = "✅ PASS" if passed else "❌ FAIL"
print(
f" accuracy={acc:.3f} threshold={config.accuracy_threshold} {status}"
)
all_results.append(
{

3
test/registered/amd/accuracy/mi35x/test_deepseek_r1_mxfp4_eval_mi35x.py
View File

@@ -239,6 +239,9 @@ class TestDeepSeekR1MXFP4EvalMI35x(unittest.TestCase):
)
passed = acc >= config.accuracy_threshold
status = "✅ PASS" if passed else "❌ FAIL"
print(
f" accuracy={acc:.3f} threshold={config.accuracy_threshold} {status}"
)
all_results.append(
{

119
test/registered/amd/accuracy/mi35x/test_deepseek_v32_dp_eval_mi35x.py Normal file
View File

@@ -0,0 +1,119 @@
"""MI35x DeepSeek-V3.2 DP GSM8K Accuracy Evaluation Test (8-GPU)
Tests DeepSeek-V3.2 with DP=8 + TP=8 + dp-attention using few-shot
completion benchmark on MI35x.
Registry: nightly-amd-accuracy-8-gpu-mi35x-deepseek-v32-dp suite
"""
import os
# Set HF cache for MI35x
os.environ.setdefault("HF_HOME", "/data2/models/huggingface")
os.environ.setdefault("HF_HUB_CACHE", "/data2/models/huggingface/hub")
import unittest
from types import SimpleNamespace
from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_amd_ci
from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k
from sglang.test.send_one import BenchArgs, send_one_prompt
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
CustomTestCase,
is_in_ci,
popen_launch_server,
write_github_step_summary,
)
# Register for AMD CI - MI35x DeepSeek-V3.2 DP accuracy test
register_amd_ci(
est_time=3600,
suite="nightly-amd-accuracy-8-gpu-mi35x-deepseek-v32-dp",
nightly=True,
)
DEEPSEEK_V32_MODEL_PATH = "deepseek-ai/DeepSeek-V3.2"
# Accuracy threshold
GSM8K_ACCURACY_THRESHOLD = 0.935
class TestDeepseekV32DP(CustomTestCase):
"""Test DeepSeek V3.2 with DP=8 + TP=8 + dp-attention.
This test runs GSM8K evaluation and measures accuracy on MI35x.
"""
@classmethod
def setUpClass(cls):
cls.model = DEEPSEEK_V32_MODEL_PATH
cls.base_url = DEFAULT_URL_FOR_TEST
other_args = [
"--trust-remote-code",
"--tp",
"8",
"--dp",
"8",
"--enable-dp-attention",
"--model-loader-extra-config",
'{"enable_multithread_load": true}',
"--nsa-prefill-backend",
"tilelang",
"--nsa-decode-backend",
"tilelang",
]
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=other_args,
)
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
def test_a_gsm8k(self):
"""GSM8K evaluation for DP configuration.
Named with 'a' prefix to run first (alphabetically) to warm up the server.
"""
args = SimpleNamespace(
num_shots=20,
data_path=None,
num_questions=1400,
parallel=1400,
max_new_tokens=512,
host="http://127.0.0.1",
port=int(self.base_url.split(":")[-1]),
)
metrics = run_eval_few_shot_gsm8k(args)
print(f"{metrics=}")
if is_in_ci():
write_github_step_summary(
f"### test_gsm8k (deepseek-v32 DP MI35x)\n"
f'{metrics["accuracy"]=:.3f}\n'
)
self.assertGreater(metrics["accuracy"], GSM8K_ACCURACY_THRESHOLD)
def test_bs_1_speed(self):
"""Single batch speed test for DP configuration."""
args = BenchArgs(port=int(self.base_url.split(":")[-1]), max_new_tokens=2048)
acc_length, speed = send_one_prompt(args)
print(f"{speed=:.2f}")
if is_in_ci():
write_github_step_summary(
f"### test_bs_1_speed (deepseek-v32 DP MI35x)\n"
f"{speed=:.2f} token/s\n"
)
self.assertGreater(speed, 10)
if __name__ == "__main__":
unittest.main()

3
test/registered/amd/accuracy/mi35x/test_deepseek_v32_eval_mi35x.py
View File

@@ -215,6 +215,9 @@ class TestDeepSeekV32EvalMI35x(unittest.TestCase):
)
passed = acc >= config.accuracy_threshold
status = "✅ PASS" if passed else "❌ FAIL"
print(
f" accuracy={acc:.3f} threshold={config.accuracy_threshold} {status}"
)
all_results.append(
{

142
test/registered/amd/accuracy/mi35x/test_deepseek_v32_mtp_eval_mi35x.py Normal file
View File

@@ -0,0 +1,142 @@
"""MI35x DeepSeek-V3.2 TP+MTP GSM8K Accuracy Evaluation Test (8-GPU)
Tests DeepSeek-V3.2 with TP=8 + MTP (EAGLE speculative decoding) using few-shot
completion benchmark on MI35x.
Registry: nightly-amd-accuracy-8-gpu-mi35x-deepseek-v32-mtp suite
"""
import os
# Set HF cache for MI35x
os.environ.setdefault("HF_HOME", "/data2/models/huggingface")
os.environ.setdefault("HF_HUB_CACHE", "/data2/models/huggingface/hub")
import unittest
from types import SimpleNamespace
import requests
from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_amd_ci
from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k
from sglang.test.send_one import BenchArgs, send_one_prompt
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
CustomTestCase,
is_in_ci,
popen_launch_server,
write_github_step_summary,
)
# Register for AMD CI - MI35x DeepSeek-V3.2 TP+MTP accuracy test
register_amd_ci(
est_time=3600,
suite="nightly-amd-accuracy-8-gpu-mi35x-deepseek-v32-mtp",
nightly=True,
)
DEEPSEEK_V32_MODEL_PATH = "deepseek-ai/DeepSeek-V3.2"
# Accuracy and performance thresholds
GSM8K_ACCURACY_THRESHOLD = 0.94
AVG_SPEC_ACCEPT_LENGTH_THRESHOLD = 2.7
class TestDeepseekV32TPMTP(CustomTestCase):
"""Test DeepSeek V3.2 with TP=8 + MTP (EAGLE speculative decoding).
This test runs GSM8K evaluation and measures both accuracy and
speculative decoding acceptance length on MI35x.
"""
@classmethod
def setUpClass(cls):
cls.model = DEEPSEEK_V32_MODEL_PATH
cls.base_url = DEFAULT_URL_FOR_TEST
other_args = [
"--trust-remote-code",
"--tp",
"8",
"--speculative-algorithm",
"EAGLE",
"--speculative-num-steps",
"3",
"--speculative-eagle-topk",
"1",
"--speculative-num-draft-tokens",
"4",
"--mem-frac",
"0.7",
"--model-loader-extra-config",
'{"enable_multithread_load": true}',
"--nsa-prefill-backend",
"tilelang",
"--nsa-decode-backend",
"tilelang",
]
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=other_args,
)
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
def test_a_gsm8k(self):
"""GSM8K evaluation for TP+MTP configuration.
Named with 'a' prefix to run first (alphabetically) to warm up the server.
"""
requests.get(self.base_url + "/flush_cache")
args = SimpleNamespace(
num_shots=20,
data_path=None,
num_questions=1400,
parallel=1400,
max_new_tokens=512,
host="http://127.0.0.1",
port=int(self.base_url.split(":")[-1]),
)
metrics = run_eval_few_shot_gsm8k(args)
print(f"{metrics=}")
server_info = requests.get(self.base_url + "/get_server_info")
avg_spec_accept_length = server_info.json()["internal_states"][0][
"avg_spec_accept_length"
]
print(f"{avg_spec_accept_length=}")
if is_in_ci():
write_github_step_summary(
f"### test_gsm8k (deepseek-v32 TP+MTP MI35x)\n"
f'{metrics["accuracy"]=:.3f}\n'
f"{avg_spec_accept_length=:.2f}\n"
)
self.assertGreater(metrics["accuracy"], GSM8K_ACCURACY_THRESHOLD)
self.assertGreater(avg_spec_accept_length, AVG_SPEC_ACCEPT_LENGTH_THRESHOLD)
def test_bs_1_speed(self):
"""Single batch speed test for TP+MTP configuration."""
args = BenchArgs(port=int(self.base_url.split(":")[-1]), max_new_tokens=2048)
acc_length, speed = send_one_prompt(args)
print(f"{acc_length=:.2f} {speed=:.2f}")
if is_in_ci():
write_github_step_summary(
f"### test_bs_1_speed (deepseek-v32 TP+MTP MI35x)\n"
f"{acc_length=:.2f}\n"
f"{speed=:.2f} token/s\n"
)
self.assertGreater(acc_length, AVG_SPEC_ACCEPT_LENGTH_THRESHOLD)
self.assertGreater(speed, 55) # Lowered from 60 for AMD MI35x
if __name__ == "__main__":
unittest.main()

3
test/registered/amd/accuracy/mi35x/test_gpt_oss_eval_mi35x.py
View File

@@ -218,6 +218,9 @@ class TestGptOssEvalMI35x(unittest.TestCase):
)
passed = acc >= config.accuracy_threshold
status = "✅ PASS" if passed else "❌ FAIL"
print(
f" accuracy={acc:.3f} threshold={config.accuracy_threshold} {status}"
)
all_results.append(
{

1
test/registered/amd/accuracy/mi35x/test_grok1_int4_eval_mi35x.py
View File

@@ -140,6 +140,7 @@ class TestGrok1INT4EvalMI35x(unittest.TestCase):
)
passed = acc >= self.accuracy_threshold
status = "✅ PASS" if passed else "❌ FAIL"
print(f" accuracy={acc:.3f} threshold={self.accuracy_threshold} {status}")
summary = f"### GROK1-INT4 (MI35x)\n\n"
summary += f"| Model | Accuracy | Threshold | Status |\n"

1
test/registered/amd/accuracy/mi35x/test_grok2_eval_mi35x.py
View File

@@ -142,6 +142,7 @@ class TestGrok2EvalMI35x(unittest.TestCase):
)
passed = acc >= self.accuracy_threshold
status = "✅ PASS" if passed else "❌ FAIL"
print(f" accuracy={acc:.3f} threshold={self.accuracy_threshold} {status}")
summary = f"### GROK2 (MI35x)\n\n"
summary += f"| Model | Accuracy | Threshold | Status |\n"

3
test/registered/amd/accuracy/test_deepseek_r1_eval_amd.py
View File

@@ -273,6 +273,9 @@ class TestDeepSeekR1EvalAMD(unittest.TestCase):
)
passed = acc >= config.accuracy_threshold
status = "✅ PASS" if passed else "❌ FAIL"
print(
f" accuracy={acc:.3f} threshold={config.accuracy_threshold} {status}"
)
all_results.append(
{

1
test/registered/amd/accuracy/test_deepseek_v31_eval_amd.py
View File

@@ -137,6 +137,7 @@ class TestDeepSeekV31EvalAMD(unittest.TestCase):
)
passed = acc >= self.accuracy_threshold
status = "✅ PASS" if passed else "❌ FAIL"
print(f" accuracy={acc:.3f} threshold={self.accuracy_threshold} {status}")
summary = f"### DeepSeek-V3.1 (MI300X)\n\n"
summary += f"| Model | Accuracy | Threshold | Status |\n"

122
test/registered/amd/accuracy/test_deepseek_v32_dp_eval_amd.py Normal file
View File

@@ -0,0 +1,122 @@
"""AMD DeepSeek-V3.2 DP GSM8K Accuracy Evaluation Test (8-GPU)
Tests DeepSeek-V3.2 with DP=8 + TP=8 + dp-attention using few-shot
completion benchmark on MI325/MI300X.
Registry: nightly-amd-accuracy-8-gpu-deepseek-v32-dp suite
"""
import os
import unittest
from types import SimpleNamespace
from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_amd_ci
from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k
from sglang.test.send_one import BenchArgs, send_one_prompt
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
CustomTestCase,
is_in_ci,
popen_launch_server,
write_github_step_summary,
)
# Register for AMD CI - DeepSeek-V3.2 DP accuracy test
register_amd_ci(
est_time=5400,
suite="nightly-amd-accuracy-8-gpu-deepseek-v32-dp",
nightly=True,
)
DEEPSEEK_V32_MODEL_PATH = "deepseek-ai/DeepSeek-V3.2"
# Accuracy threshold
GSM8K_ACCURACY_THRESHOLD = 0.935
class TestDeepseekV32DP(CustomTestCase):
"""Test DeepSeek V3.2 with DP=8 + TP=8 + dp-attention.
This test runs GSM8K evaluation and measures accuracy on MI325/MI300X.
"""
@classmethod
def setUpClass(cls):
cls.model = DEEPSEEK_V32_MODEL_PATH
cls.base_url = DEFAULT_URL_FOR_TEST
other_args = [
"--trust-remote-code",
"--tp",
"8",
"--dp",
"8",
"--enable-dp-attention",
"--attention-backend",
"aiter",
"--chunked-prefill-size",
"131072",
"--mem-fraction-static",
"0.85",
"--model-loader-extra-config",
'{"enable_multithread_load": true}',
"--watchdog-timeout",
"1200",
]
env = os.environ.copy()
env["SGLANG_USE_AITER"] = "1"
env["SGLANG_USE_ROCM700A"] = "1"
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=other_args,
env=env,
)
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
def test_a_gsm8k(self):
"""GSM8K evaluation for DP configuration.
Named with 'a' prefix to run first (alphabetically) to warm up the server.
"""
args = SimpleNamespace(
num_shots=20,
data_path=None,
num_questions=1400,
parallel=1400,
max_new_tokens=512,
host="http://127.0.0.1",
port=int(self.base_url.split(":")[-1]),
)
metrics = run_eval_few_shot_gsm8k(args)
print(f"{metrics=}")
if is_in_ci():
write_github_step_summary(
f"### test_gsm8k (deepseek-v32 DP MI325)\n"
f'{metrics["accuracy"]=:.3f}\n'
)
self.assertGreater(metrics["accuracy"], GSM8K_ACCURACY_THRESHOLD)
def test_bs_1_speed(self):
"""Single batch speed test for DP configuration."""
args = BenchArgs(port=int(self.base_url.split(":")[-1]), max_new_tokens=2048)
acc_length, speed = send_one_prompt(args)
print(f"{speed=:.2f}")
if is_in_ci():
write_github_step_summary(
f"### test_bs_1_speed (deepseek-v32 DP MI325)\n"
f"{speed=:.2f} token/s\n"
)
self.assertGreater(speed, 10)
if __name__ == "__main__":
unittest.main()

248
test/registered/amd/accuracy/test_deepseek_v32_eval_amd.py Normal file
View File

@@ -0,0 +1,248 @@
"""AMD DeepSeek-V3.2 GSM8K Completion Evaluation Test (8-GPU)
Tests DeepSeek-V3.2 with basic configuration using few-shot completion
benchmark on MI325/MI300X.
Registry: nightly-amd-accuracy-8-gpu-deepseek-v32 suite
"""
import ast
import os
import re
import time
import unittest
from dataclasses import dataclass
from typing import List, Optional, Tuple
import numpy as np
from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_amd_ci
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
is_in_ci,
popen_launch_server,
write_github_step_summary,
)
from sglang.utils import download_and_cache_file, read_jsonl
# Register for AMD CI - DeepSeek-V3.2 accuracy test (~60 min for basic only)
register_amd_ci(
est_time=3600,
suite="nightly-amd-accuracy-8-gpu-deepseek-v32",
nightly=True,
)
INVALID = -9999999
@dataclass
class ModelConfig:
"""Configuration for a model to test."""
model_path: str
tp_size: int = 8
accuracy_threshold: float = 0.50
other_args: Optional[List[str]] = None
env_vars: Optional[dict] = None
timeout: Optional[int] = None
variant: Optional[str] = None
def __post_init__(self):
if self.other_args is None:
self.other_args = []
if self.env_vars is None:
self.env_vars = {}
def get_display_name(self) -> str:
if self.variant:
return f"{self.model_path} ({self.variant})"
return self.model_path
# DeepSeek-V3.2 models for MI325/MI300X - basic variant
DEEPSEEK_V32_MODELS = [
# DeepSeek-V3.2 basic (TP=8 only)
ModelConfig(
model_path="deepseek-ai/DeepSeek-V3.2",
tp_size=8,
accuracy_threshold=0.93,
timeout=3600,
variant="basic",
other_args=[
"--trust-remote-code",
"--attention-backend",
"aiter",
"--chunked-prefill-size",
"131072",
"--mem-fraction-static",
"0.85",
"--model-loader-extra-config",
'{"enable_multithread_load": true}',
"--watchdog-timeout",
"1200", # 20 minutes for weight loading
],
env_vars={"SGLANG_USE_AITER": "1"},
),
]
def get_one_example(lines, i, include_answer):
"""Format a single GSM8K example."""
ret = "Question: " + lines[i]["question"] + "\nAnswer:"
if include_answer:
ret += " " + lines[i]["answer"]
return ret
def get_few_shot_examples(lines, k):
"""Get k few-shot examples for prompting."""
ret = ""
for i in range(k):
ret += get_one_example(lines, i, True) + "\n\n"
return ret
def get_answer_value(answer_str):
"""Extract numerical answer from response."""
answer_str = answer_str.replace(",", "")
numbers = re.findall(r"\d+", answer_str)
if len(numbers) < 1:
return INVALID
try:
return ast.literal_eval(numbers[-1])
except SyntaxError:
return INVALID
def run_gsm8k_benchmark(
base_url: str,
num_questions: int = 200,
num_shots: int = 5,
parallel: int = 64,
) -> Tuple[float, float, float]:
"""Run GSM8K few-shot completion benchmark."""
import sglang as sgl
from sglang.lang.backend.runtime_endpoint import RuntimeEndpoint
url = "https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/test.jsonl"
data_path = download_and_cache_file(url)
lines = list(read_jsonl(data_path))
few_shot_examples = get_few_shot_examples(lines, num_shots)
questions = []
labels = []
for i in range(len(lines[:num_questions])):
questions.append(get_one_example(lines, i, False))
labels.append(get_answer_value(lines[i]["answer"]))
assert all(l != INVALID for l in labels)
arguments = [{"question": q} for q in questions]
@sgl.function
def few_shot_gsm8k(s, question):
s += few_shot_examples + question
s += sgl.gen(
"answer", max_tokens=512, stop=["Question", "Assistant:", "<|separator|>"]
)
backend = RuntimeEndpoint(base_url)
sgl.set_default_backend(backend)
tic = time.perf_counter()
states = few_shot_gsm8k.run_batch(
arguments, temperature=0, num_threads=parallel, progress_bar=True
)
latency = time.perf_counter() - tic
preds = [get_answer_value(states[i]["answer"]) for i in range(len(states))]
acc = np.mean(np.array(preds) == np.array(labels))
invalid = np.mean(np.array(preds) == INVALID)
return float(acc), float(invalid), float(latency)
class TestDeepSeekV32EvalAMD(unittest.TestCase):
"""DeepSeek-V3.2 GSM8K Completion Evaluation Test for AMD MI325/MI300X."""
@classmethod
def setUpClass(cls):
cls.models = DEEPSEEK_V32_MODELS
cls.base_url = DEFAULT_URL_FOR_TEST
cls.num_questions = int(os.environ.get("GSM8K_NUM_QUESTIONS", "200"))
def test_deepseek_v32_accuracy(self):
"""Test DeepSeek-V3.2 models with GSM8K completion benchmark."""
all_results = []
summary = "### DeepSeek-V3.2 Models (MI325)\n\n"
summary += "| Model | Variant | TP | Accuracy | Threshold | Status |\n"
summary += "| ----- | ------- | -- | -------- | --------- | ------ |\n"
for config in self.models:
display_name = config.get_display_name()
with self.subTest(model=display_name):
print(f"\n{'='*60}")
print(f"Testing: {display_name}")
print(f"{'='*60}")
env = os.environ.copy()
for key, value in config.env_vars.items():
env[key] = value
other_args = list(config.other_args)
other_args.extend(["--tp", str(config.tp_size)])
timeout = config.timeout or DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH
try:
process = popen_launch_server(
model=config.model_path,
base_url=self.base_url,
timeout=timeout,
other_args=other_args,
env=env,
)
try:
acc, invalid, latency = run_gsm8k_benchmark(
self.base_url, num_questions=self.num_questions
)
passed = acc >= config.accuracy_threshold
status = "✅ PASS" if passed else "❌ FAIL"
print(
f" accuracy={acc:.3f} threshold={config.accuracy_threshold} {status}"
)
all_results.append(
{
"model": display_name,
"accuracy": acc,
"passed": passed,
}
)
summary += f"| {config.model_path} | {config.variant or 'N/A'} | {config.tp_size} | {acc:.3f} | {config.accuracy_threshold} | {status} |\n"
finally:
kill_process_tree(process.pid)
except Exception as e:
summary += f"| {config.model_path} | {config.variant or 'N/A'} | {config.tp_size} | N/A | {config.accuracy_threshold} | ❌ ERROR |\n"
all_results.append(
{
"model": display_name,
"accuracy": None,
"passed": False,
"error": str(e),
}
)
if is_in_ci():
write_github_step_summary(summary)
failed = [r for r in all_results if not r["passed"]]
if failed:
raise AssertionError(f"Failed models: {[r['model'] for r in failed]}")
if __name__ == "__main__":
unittest.main()

142
test/registered/amd/accuracy/test_deepseek_v32_mtp_eval_amd.py Normal file
View File

@@ -0,0 +1,142 @@
"""AMD DeepSeek-V3.2 TP+MTP GSM8K Accuracy Evaluation Test (8-GPU)
Tests DeepSeek-V3.2 with TP=8 + MTP (EAGLE speculative decoding) using few-shot
completion benchmark on MI325/MI300X.
Registry: nightly-amd-accuracy-8-gpu-deepseek-v32-mtp suite
"""
import os
import unittest
from types import SimpleNamespace
import requests
from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_amd_ci
from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k
from sglang.test.send_one import BenchArgs, send_one_prompt
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
CustomTestCase,
is_in_ci,
popen_launch_server,
write_github_step_summary,
)
# Register for AMD CI - DeepSeek-V3.2 TP+MTP accuracy test
register_amd_ci(
est_time=3600,
suite="nightly-amd-accuracy-8-gpu-deepseek-v32-mtp",
nightly=True,
)
DEEPSEEK_V32_MODEL_PATH = "deepseek-ai/DeepSeek-V3.2"
# Accuracy and performance thresholds
GSM8K_ACCURACY_THRESHOLD = 0.94
AVG_SPEC_ACCEPT_LENGTH_THRESHOLD = 2.7
class TestDeepseekV32TPMTP(CustomTestCase):
"""Test DeepSeek V3.2 with TP=8 + MTP (EAGLE speculative decoding).
This test runs GSM8K evaluation and measures both accuracy and
speculative decoding acceptance length on MI325/MI300X.
"""
@classmethod
def setUpClass(cls):
cls.model = DEEPSEEK_V32_MODEL_PATH
cls.base_url = DEFAULT_URL_FOR_TEST
other_args = [
"--trust-remote-code",
"--tp",
"8",
"--attention-backend",
"aiter",
"--chunked-prefill-size",
"131072",
"--speculative-algorithm",
"EAGLE",
"--speculative-num-steps",
"3",
"--speculative-eagle-topk",
"1",
"--speculative-num-draft-tokens",
"4",
"--mem-fraction-static",
"0.7",
"--model-loader-extra-config",
'{"enable_multithread_load": true}',
"--watchdog-timeout",
"1200",
]
env = os.environ.copy()
env["SGLANG_USE_AITER"] = "1"
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=other_args,
env=env,
)
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
def test_a_gsm8k(self):
"""GSM8K evaluation for TP+MTP configuration.
Named with 'a' prefix to run first (alphabetically) to warm up the server.
"""
requests.get(self.base_url + "/flush_cache")
args = SimpleNamespace(
num_shots=20,
data_path=None,
num_questions=1400,
parallel=1400,
max_new_tokens=512,
host="http://127.0.0.1",
port=int(self.base_url.split(":")[-1]),
)
metrics = run_eval_few_shot_gsm8k(args)
print(f"{metrics=}")
server_info = requests.get(self.base_url + "/get_server_info")
avg_spec_accept_length = server_info.json()["internal_states"][0][
"avg_spec_accept_length"
]
print(f"{avg_spec_accept_length=}")
if is_in_ci():
write_github_step_summary(
f"### test_gsm8k (deepseek-v32 TP+MTP MI325)\n"
f'{metrics["accuracy"]=:.3f}\n'
f"{avg_spec_accept_length=:.2f}\n"
)
self.assertGreater(metrics["accuracy"], GSM8K_ACCURACY_THRESHOLD)
self.assertGreater(avg_spec_accept_length, AVG_SPEC_ACCEPT_LENGTH_THRESHOLD)
def test_bs_1_speed(self):
"""Single batch speed test for TP+MTP configuration."""
args = BenchArgs(port=int(self.base_url.split(":")[-1]), max_new_tokens=2048)
acc_length, speed = send_one_prompt(args)
print(f"{acc_length=:.2f} {speed=:.2f}")
if is_in_ci():
write_github_step_summary(
f"### test_bs_1_speed (deepseek-v32 TP+MTP MI325)\n"
f"{acc_length=:.2f}\n"
f"{speed=:.2f} token/s\n"
)
self.assertGreater(acc_length, AVG_SPEC_ACCEPT_LENGTH_THRESHOLD)
self.assertGreater(speed, 55) # Lowered from 60 for AMD MI325
if __name__ == "__main__":
unittest.main()

123
test/registered/amd/accuracy/test_deepseek_v32_tc_eval_amd.py Normal file
View File

@@ -0,0 +1,123 @@
"""AMD DeepSeek-V3.2 TC GSM8K Accuracy Evaluation Test (8-GPU)
Tests DeepSeek-V3.2 with Torch Compile configuration using few-shot
completion benchmark on MI325/MI300X.
Registry: nightly-amd-accuracy-8-gpu-deepseek-v32-tc suite
"""
import os
import unittest
from types import SimpleNamespace
from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_amd_ci
from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k
from sglang.test.send_one import BenchArgs, send_one_prompt
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
CustomTestCase,
is_in_ci,
popen_launch_server,
write_github_step_summary,
)
# Register for AMD CI - DeepSeek-V3.2 TC accuracy test
register_amd_ci(
est_time=7200,
suite="nightly-amd-accuracy-8-gpu-deepseek-v32-tc",
nightly=True,
)
DEEPSEEK_V32_MODEL_PATH = "deepseek-ai/DeepSeek-V3.2"
# Accuracy threshold
GSM8K_ACCURACY_THRESHOLD = 0.935
class TestDeepseekV32TC(CustomTestCase):
"""Test DeepSeek V3.2 with Torch Compile.
This test runs GSM8K evaluation and measures accuracy on MI325/MI300X.
"""
@classmethod
def setUpClass(cls):
cls.model = DEEPSEEK_V32_MODEL_PATH
cls.base_url = DEFAULT_URL_FOR_TEST
other_args = [
"--trust-remote-code",
"--tp",
"8",
"--attention-backend",
"aiter",
"--chunked-prefill-size",
"131072",
"--mem-fraction-static",
"0.70",
"--cuda-graph-max-bs",
"8",
"--enable-torch-compile",
"--disable-cuda-graph",
"--model-loader-extra-config",
'{"enable_multithread_load": true}',
"--watchdog-timeout",
"1200",
]
env = os.environ.copy()
env["SGLANG_USE_AITER"] = "1"
env["SGLANG_USE_ROCM700A"] = "1"
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=other_args,
env=env,
)
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
def test_a_gsm8k(self):
"""GSM8K evaluation for TC configuration.
Named with 'a' prefix to run first (alphabetically) to warm up the server.
"""
args = SimpleNamespace(
num_shots=20,
data_path=None,
num_questions=1400,
parallel=1400,
max_new_tokens=512,
host="http://127.0.0.1",
port=int(self.base_url.split(":")[-1]),
)
metrics = run_eval_few_shot_gsm8k(args)
print(f"{metrics=}")
if is_in_ci():
write_github_step_summary(
f"### test_gsm8k (deepseek-v32 TC MI325)\n"
f'{metrics["accuracy"]=:.3f}\n'
)
self.assertGreater(metrics["accuracy"], GSM8K_ACCURACY_THRESHOLD)
def test_bs_1_speed(self):
"""Single batch speed test for TC configuration."""
args = BenchArgs(port=int(self.base_url.split(":")[-1]), max_new_tokens=2048)
acc_length, speed = send_one_prompt(args)
print(f"{speed=:.2f}")
if is_in_ci():
write_github_step_summary(
f"### test_bs_1_speed (deepseek-v32 TC MI325)\n"
f"{speed=:.2f} token/s\n"
)
self.assertGreater(speed, 10)
if __name__ == "__main__":
unittest.main()

5
test/registered/amd/accuracy/test_gpt_oss_eval_amd.py
View File

@@ -56,7 +56,7 @@ GPT_OSS_MODELS = [
ModelConfig(
model_path="lmsys/gpt-oss-20b-bf16",
tp_size=8,
accuracy_threshold=0.47,
accuracy_threshold=0.45,
other_args=[
"--chunked-prefill-size",
"130172",
@@ -211,6 +211,9 @@ class TestGptOssEvalAMD(unittest.TestCase):
)
passed = acc >= config.accuracy_threshold
status = "✅ PASS" if passed else "❌ FAIL"
print(
f" accuracy={acc:.3f} threshold={config.accuracy_threshold} {status}"
)
all_results.append(
{

1
test/registered/amd/accuracy/test_grok1_fp8_eval_amd.py
View File

@@ -140,6 +140,7 @@ class TestGrok1FP8EvalAMD(unittest.TestCase):
)
passed = acc >= self.accuracy_threshold
status = "✅ PASS" if passed else "❌ FAIL"
print(f" accuracy={acc:.3f} threshold={self.accuracy_threshold} {status}")
summary = f"### GROK1-FP8 (MI300X)\n\n"
summary += f"| Model | Accuracy | Threshold | Status |\n"

1
test/registered/amd/accuracy/test_grok1_int4_eval_amd.py
View File

@@ -140,6 +140,7 @@ class TestGrok1INT4EvalAMD(unittest.TestCase):
)
passed = acc >= self.accuracy_threshold
status = "✅ PASS" if passed else "❌ FAIL"
print(f" accuracy={acc:.3f} threshold={self.accuracy_threshold} {status}")
summary = f"### GROK1-INT4 (MI300X)\n\n"
summary += f"| Model | Accuracy | Threshold | Status |\n"

1
test/registered/amd/accuracy/test_grok2_eval_amd.py
View File

@@ -140,6 +140,7 @@ class TestGrok2EvalAMD(unittest.TestCase):
)
passed = acc >= self.accuracy_threshold
status = "✅ PASS" if passed else "❌ FAIL"
print(f" accuracy={acc:.3f} threshold={self.accuracy_threshold} {status}")
summary = f"### GROK2 (MI300X)\n\n"
summary += f"| Model | Accuracy | Threshold | Status |\n"

3
test/registered/amd/accuracy/test_grok_eval_amd.py
View File

@@ -251,6 +251,9 @@ class TestGrokEvalAMD(unittest.TestCase):
)
passed = acc >= config.accuracy_threshold
status = "✅ PASS" if passed else "❌ FAIL"
print(
f" accuracy={acc:.3f} threshold={config.accuracy_threshold} {status}"
)
all_results.append(
{

6
test/registered/amd/accuracy/test_gsm8k_eval_amd.py
View File

@@ -59,9 +59,9 @@ MODEL_SCORE_THRESHOLDS = {
"neuralmagic/Meta-Llama-3.1-8B-Instruct-FP8": 0.8,
"neuralmagic/Mistral-7B-Instruct-v0.3-FP8": 0.54,
"neuralmagic/Meta-Llama-3.1-70B-Instruct-FP8": 0.94,
"neuralmagic/Qwen2-72B-Instruct-FP8": 0.94,
"neuralmagic/Qwen2-57B-A14B-Instruct-FP8": 0.86,
"neuralmagic/Mixtral-8x7B-Instruct-v0.1-FP8": 0.62,
"neuralmagic/Qwen2-72B-Instruct-FP8": 0.92,
"neuralmagic/Qwen2-57B-A14B-Instruct-FP8": 0.81,
"neuralmagic/Mixtral-8x7B-Instruct-v0.1-FP8": 0.57,
"neuralmagic/DeepSeek-Coder-V2-Lite-Instruct-FP8": 0.84,
}

101
test/registered/amd/accuracy/test_kimi_k2_eval_amd.py Normal file
View File

@@ -0,0 +1,101 @@
"""AMD Kimi-K2 GSM8K Completion Evaluation Test (8-GPU)
Tests moonshotai/Kimi-K2-Instruct-0905 with GSM8K few-shot benchmark on MI325.
Registry: nightly-amd-accuracy-8-gpu-kimi-k2 suite
"""
import os
import unittest
from types import SimpleNamespace
import requests
from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_amd_ci
from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k
from sglang.test.test_utils import (
DEFAULT_URL_FOR_TEST,
CustomTestCase,
is_in_ci,
popen_launch_server,
write_github_step_summary,
)
# Register for AMD CI - Kimi K2 accuracy test (~60 min)
register_amd_ci(est_time=3600, suite="nightly-amd-accuracy-8-gpu-kimi-k2", nightly=True)
KIMI_K2_MODEL_PATH = "moonshotai/Kimi-K2-Instruct-0905"
SERVER_LAUNCH_TIMEOUT = 3600
ACCURACY_THRESHOLD = 0.94
class TestKimiK2EvalAMD(CustomTestCase):
"""Kimi-K2 GSM8K Completion Evaluation Test for AMD MI325."""
@classmethod
def setUpClass(cls):
cls.model = KIMI_K2_MODEL_PATH
cls.base_url = DEFAULT_URL_FOR_TEST
other_args = [
"--tp",
"8",
"--decode-attention-backend",
"triton",
"--prefill-attention-backend",
"aiter",
"--trust-remote-code",
"--model-loader-extra-config",
'{"enable_multithread_load": true}',
]
env = os.environ.copy()
env["SGLANG_USE_AITER"] = "1"
env["SGLANG_ROCM_FUSED_DECODE_MLA"] = "0"
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=SERVER_LAUNCH_TIMEOUT,
other_args=other_args,
env=env,
)
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
def test_kimi_k2_gsm8k_accuracy(self):
"""Test Kimi-K2 with GSM8K few-shot completion benchmark."""
requests.get(self.base_url + "/flush_cache")
args = SimpleNamespace(
num_shots=8,
data_path=None,
num_questions=1319,
parallel=1319,
max_new_tokens=512,
host="http://127.0.0.1",
port=int(self.base_url.split(":")[-1]),
)
metrics = run_eval_few_shot_gsm8k(args)
acc = metrics["accuracy"]
passed = acc >= ACCURACY_THRESHOLD
status = "✅ PASS" if passed else "❌ FAIL"
print(f" accuracy={acc:.3f} threshold={ACCURACY_THRESHOLD} {status}")
if is_in_ci():
summary = "### Kimi-K2 Model (MI325)\n\n"
summary += "| Model | TP | Accuracy | Threshold | Status |\n"
summary += "| ----- | -- | -------- | --------- | ------ |\n"
summary += f"| {KIMI_K2_MODEL_PATH} | 8 | {acc:.3f} | {ACCURACY_THRESHOLD} | {status} |\n"
write_github_step_summary(summary)
self.assertGreaterEqual(
acc,
ACCURACY_THRESHOLD,
f"Kimi-K2 accuracy {acc:.3f} below threshold {ACCURACY_THRESHOLD}",
)
if __name__ == "__main__":
unittest.main()

3
test/registered/amd/accuracy/test_vlms_mmmu_eval_amd.py
View File

@@ -127,7 +127,8 @@ TRITON_ATTENTION_MODELS = {
# Models known to fail on AMD - exclude from testing
AMD_FAILING_VLM_MODELS = {
# Add models here as they are discovered to fail
# GLM-4.1V processor not registered yet (Glm4vForConditionalGeneration)
"zai-org/GLM-4.1V-9B-Thinking",
}

2
test/registered/amd/perf/mi35x/test_deepseek_v32_basic_perf_mi35x.py
View File

@@ -93,6 +93,8 @@ class TestNightlyDeepseekV32BasicPerformance(unittest.TestCase):
"0.85",
"--model-loader-extra-config",
'{"enable_multithread_load": true}',
"--watchdog-timeout",
"1200",
],
}

62
test/registered/amd/perf/mi35x/test_deepseek_v32_mtp_perf_mi35x.py
View File

@@ -13,12 +13,17 @@ Example usage:
import os
import unittest
from typing import List
from typing import List, Optional, Tuple
from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_amd_ci
from sglang.test.nightly_bench_utils import BenchmarkResult
from sglang.test.nightly_utils import NightlyBenchmarkRunner
from sglang.test.test_utils import DEFAULT_URL_FOR_TEST, _parse_int_list_env
from sglang.test.test_utils import (
DEFAULT_URL_FOR_TEST,
_parse_int_list_env,
popen_launch_server,
)
# Register for AMD CI - DeepSeek-V3.2 MTP benchmark (~90 min)
register_amd_ci(
@@ -57,11 +62,58 @@ def generate_simple_markdown_report(results: List[BenchmarkResult]) -> str:
return summary
def _run_benchmark_with_timeout(
runner: NightlyBenchmarkRunner,
model_path: str,
batch_sizes: List[int],
input_lens: Tuple[int, ...],
output_lens: Tuple[int, ...],
other_args: List[str],
variant: str,
extra_bench_args: Optional[List[str]],
timeout: int,
) -> Tuple[List[BenchmarkResult], bool, Optional[float]]:
"""Run benchmark with a custom server launch timeout."""
model_description = f"{model_path}" + (f" ({variant})" if variant else "")
process = popen_launch_server(
model=model_path,
base_url=runner.base_url,
other_args=other_args,
timeout=timeout,
)
try:
profile_path_prefix, json_output_file = runner.generate_profile_filename(
model_path, variant
)
bench_args = list(extra_bench_args) if extra_bench_args else []
if variant:
bench_args.extend(["--run-name", variant])
command = runner.build_benchmark_command(
model_path,
batch_sizes,
input_lens,
output_lens,
profile_path_prefix,
json_output_file,
extra_args=bench_args,
)
_, cmd_success = runner.run_benchmark_command(command, model_description)
if not cmd_success:
return [], False, None
benchmark_results, load_success = runner.load_benchmark_results(
json_output_file, model_description
)
return benchmark_results, load_success, None
finally:
kill_process_tree(process.pid)
# Model path can be overridden via environment variable
DEEPSEEK_V32_MODEL_PATH = os.environ.get(
"DEEPSEEK_V32_MODEL_PATH", "deepseek-ai/DeepSeek-V3.2"
)
PROFILE_DIR = "performance_profiles_deepseek_v32_mtp"
SERVER_LAUNCH_TIMEOUT = 5400
class TestNightlyDeepseekV32MTPPerformance(unittest.TestCase):
@@ -102,6 +154,8 @@ class TestNightlyDeepseekV32MTPPerformance(unittest.TestCase):
"0.7",
"--model-loader-extra-config",
'{"enable_multithread_load": true}',
"--watchdog-timeout",
"1200",
],
}
@@ -113,7 +167,8 @@ class TestNightlyDeepseekV32MTPPerformance(unittest.TestCase):
def test_bench_one_batch(self):
"""Run benchmark for MTP variant."""
try:
result_tuple = self.runner.run_benchmark_for_model(
result_tuple = _run_benchmark_with_timeout(
runner=self.runner,
model_path=self.model,
batch_sizes=self.batch_sizes,
input_lens=self.input_lens,
@@ -121,6 +176,7 @@ class TestNightlyDeepseekV32MTPPerformance(unittest.TestCase):
other_args=self.variant_config["other_args"],
variant=self.variant_config["name"],
extra_bench_args=["--trust-remote-code"],
timeout=SERVER_LAUNCH_TIMEOUT,
)
results = result_tuple[0]
success = result_tuple[1]

142
test/registered/amd/perf/test_deepseek_v32_basic_perf_amd.py Normal file
View File

@@ -0,0 +1,142 @@
"""AMD Nightly performance benchmark for DeepSeek-V3.2 model (basic variant).
This test benchmarks the DeepSeek-V3.2 model with basic TP=8 configuration on 8 GPUs.
The model path can be configured via DEEPSEEK_V32_MODEL_PATH environment variable.
Registry: nightly-perf-8-gpu-deepseek-v32-basic suite
Example usage:
DEEPSEEK_V32_MODEL_PATH=deepseek-ai/DeepSeek-V3.2 python -m pytest test_deepseek_v32_basic_perf_amd.py -v
"""
import os
import unittest
from typing import List
from sglang.test.ci.ci_register import register_amd_ci
from sglang.test.nightly_bench_utils import BenchmarkResult
from sglang.test.nightly_utils import NightlyBenchmarkRunner
from sglang.test.test_utils import DEFAULT_URL_FOR_TEST, _parse_int_list_env
# Register for AMD CI - DeepSeek-V3.2 basic benchmark (~90 min)
register_amd_ci(
est_time=5400, suite="nightly-perf-8-gpu-deepseek-v32-basic", nightly=True
)
def generate_simple_markdown_report(results: List[BenchmarkResult]) -> str:
"""Generate a simplified markdown report without traces and cost columns.
Skips the first result if it's a warmup run (duplicate batch_size).
"""
model_header = results[0].model_path
if results[0].run_name and results[0].run_name != "default":
model_header += f" ({results[0].run_name})"
gpu_config = os.getenv("GPU_CONFIG", "MI325")
if gpu_config:
model_header += f" [{gpu_config}]"
summary = f"### {model_header}\n"
summary += "| batch size | input len | latency (s) | input throughput (tok/s) | output throughput (tok/s) | ITL (ms) |\n"
summary += "| ---------- | --------- | ----------- | ------------------------ | ------------------------- | -------- |\n"
# Skip first result if it's a warmup (same batch_size as second result)
report_results = (
results[1:]
if len(results) > 1 and results[0].batch_size == results[1].batch_size
else results
)
for result in report_results:
itl = 1 / (result.output_throughput / result.batch_size) * 1000
summary += f"| {result.batch_size} | {result.input_len} | {result.latency:.2f} | {result.input_throughput:.2f} | {result.output_throughput:.2f} | {itl:.2f} |\n"
return summary
# Model path can be overridden via environment variable
DEEPSEEK_V32_MODEL_PATH = os.environ.get(
"DEEPSEEK_V32_MODEL_PATH", "deepseek-ai/DeepSeek-V3.2"
)
PROFILE_DIR = "performance_profiles_deepseek_v32_basic_mi325"
class TestNightlyDeepseekV32BasicPerformance(unittest.TestCase):
"""AMD Nightly performance benchmark for DeepSeek-V3.2 model (basic variant).
Tests the DeepSeek-V3.2 model with basic TP=8 configuration on MI325/MI300X.
"""
@classmethod
def setUpClass(cls):
cls.model = DEEPSEEK_V32_MODEL_PATH
cls.base_url = DEFAULT_URL_FOR_TEST
cls.batch_sizes = [1, 8, 16, 64]
cls.input_lens = tuple(_parse_int_list_env("NIGHTLY_INPUT_LENS", "4096"))
cls.output_lens = tuple(_parse_int_list_env("NIGHTLY_OUTPUT_LENS", "512"))
# Basic variant configuration for DeepSeek-V3.2
# MI325 uses aiter attention backend
cls.variant_config = {
"name": "basic",
"other_args": [
"--trust-remote-code",
"--tp",
"8",
"--attention-backend",
"aiter",
"--chunked-prefill-size",
"131072",
"--mem-fraction-static",
"0.85",
"--model-loader-extra-config",
'{"enable_multithread_load": true}',
"--watchdog-timeout",
"1200",
],
"env_vars": {"SGLANG_USE_AITER": "1"},
}
cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url)
cls.runner.setup_profile_directory()
# Override full_report to remove traces help text
cls.runner.full_report = f"## {cls.__name__}\n"
def test_bench_one_batch(self):
"""Run benchmark for basic variant."""
try:
result_tuple = self.runner.run_benchmark_for_model(
model_path=self.model,
batch_sizes=self.batch_sizes,
input_lens=self.input_lens,
output_lens=self.output_lens,
other_args=self.variant_config["other_args"],
variant=self.variant_config["name"],
extra_bench_args=["--trust-remote-code"],
)
results = result_tuple[0]
success = result_tuple[1]
avg_spec_accept_length = result_tuple[2] if len(result_tuple) > 2 else None
# Log speculative decoding accept length
if avg_spec_accept_length is not None:
print(f" avg_spec_accept_length={avg_spec_accept_length:.2f}")
# Use simplified report format without traces
if results:
self.runner.full_report += (
generate_simple_markdown_report(results) + "\n"
)
if not success:
raise AssertionError(
f"Benchmark failed for {self.model} (basic variant)"
)
finally:
self.runner.write_final_report()
if __name__ == "__main__":
unittest.main()

149
test/registered/amd/perf/test_deepseek_v32_mtp_perf_amd.py Normal file
View File

@@ -0,0 +1,149 @@
"""AMD Nightly performance benchmark for DeepSeek-V3.2 model (MTP variant).
This test benchmarks the DeepSeek-V3.2 model with MTP (EAGLE speculative decoding)
configuration on 8 GPUs.
The model path can be configured via DEEPSEEK_V32_MODEL_PATH environment variable.
Registry: nightly-perf-8-gpu-deepseek-v32-mtp suite
Example usage:
DEEPSEEK_V32_MODEL_PATH=deepseek-ai/DeepSeek-V3.2 python -m pytest test_deepseek_v32_mtp_perf_amd.py -v
"""
import os
import unittest
from typing import List
from sglang.test.ci.ci_register import register_amd_ci
from sglang.test.nightly_bench_utils import BenchmarkResult
from sglang.test.nightly_utils import NightlyBenchmarkRunner
from sglang.test.test_utils import DEFAULT_URL_FOR_TEST, _parse_int_list_env
# Register for AMD CI - DeepSeek-V3.2 MTP benchmark (~120 min)
register_amd_ci(
est_time=7200, suite="nightly-perf-8-gpu-deepseek-v32-mtp", nightly=True
)
def generate_simple_markdown_report(results: List[BenchmarkResult]) -> str:
"""Generate a simplified markdown report without traces and cost columns.
Skips the first result if it's a warmup run (duplicate batch_size).
"""
model_header = results[0].model_path
if results[0].run_name and results[0].run_name != "default":
model_header += f" ({results[0].run_name})"
gpu_config = os.getenv("GPU_CONFIG", "MI325")
if gpu_config:
model_header += f" [{gpu_config}]"
summary = f"### {model_header}\n"
summary += "| batch size | input len | latency (s) | input throughput (tok/s) | output throughput (tok/s) | ITL (ms) |\n"
summary += "| ---------- | --------- | ----------- | ------------------------ | ------------------------- | -------- |\n"
# Skip first result if it's a warmup (same batch_size as second result)
report_results = (
results[1:]
if len(results) > 1 and results[0].batch_size == results[1].batch_size
else results
)
for result in report_results:
itl = 1 / (result.output_throughput / result.batch_size) * 1000
summary += f"| {result.batch_size} | {result.input_len} | {result.latency:.2f} | {result.input_throughput:.2f} | {result.output_throughput:.2f} | {itl:.2f} |\n"
return summary
# Model path can be overridden via environment variable
DEEPSEEK_V32_MODEL_PATH = os.environ.get(
"DEEPSEEK_V32_MODEL_PATH", "deepseek-ai/DeepSeek-V3.2"
)
PROFILE_DIR = "performance_profiles_deepseek_v32_mtp_mi325"
class TestNightlyDeepseekV32MTPPerformance(unittest.TestCase):
"""AMD Nightly performance benchmark for DeepSeek-V3.2 model (MTP variant).
Tests the DeepSeek-V3.2 model with MTP (EAGLE speculative decoding) on MI325/MI300X.
"""
@classmethod
def setUpClass(cls):
cls.model = DEEPSEEK_V32_MODEL_PATH
cls.base_url = DEFAULT_URL_FOR_TEST
cls.batch_sizes = [1, 8, 16, 64]
cls.input_lens = tuple(_parse_int_list_env("NIGHTLY_INPUT_LENS", "4096"))
cls.output_lens = tuple(_parse_int_list_env("NIGHTLY_OUTPUT_LENS", "512"))
# MTP variant configuration for DeepSeek-V3.2
# MI325 uses aiter attention backend + EAGLE speculative decoding
cls.variant_config = {
"name": "mtp",
"other_args": [
"--trust-remote-code",
"--tp",
"8",
"--attention-backend",
"aiter",
"--chunked-prefill-size",
"131072",
"--speculative-algorithm",
"EAGLE",
"--speculative-num-steps",
"3",
"--speculative-eagle-topk",
"1",
"--speculative-num-draft-tokens",
"4",
"--mem-fraction-static",
"0.7",
"--model-loader-extra-config",
'{"enable_multithread_load": true}',
"--watchdog-timeout",
"1200",
],
"env_vars": {"SGLANG_USE_AITER": "1"},
}
cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url)
cls.runner.setup_profile_directory()
# Override full_report to remove traces help text
cls.runner.full_report = f"## {cls.__name__}\n"
def test_bench_one_batch(self):
"""Run benchmark for MTP variant."""
try:
result_tuple = self.runner.run_benchmark_for_model(
model_path=self.model,
batch_sizes=self.batch_sizes,
input_lens=self.input_lens,
output_lens=self.output_lens,
other_args=self.variant_config["other_args"],
variant=self.variant_config["name"],
extra_bench_args=["--trust-remote-code"],
)
results = result_tuple[0]
success = result_tuple[1]
avg_spec_accept_length = result_tuple[2] if len(result_tuple) > 2 else None
# Log speculative decoding accept length
if avg_spec_accept_length is not None:
print(f" avg_spec_accept_length={avg_spec_accept_length:.2f}")
# Use simplified report format without traces
if results:
self.runner.full_report += (
generate_simple_markdown_report(results) + "\n"
)
if not success:
raise AssertionError(f"Benchmark failed for {self.model} (MTP variant)")
finally:
self.runner.write_final_report()
if __name__ == "__main__":
unittest.main()