[AMD] Add AMD CI registration (1-gpu unit test) to nightly CI. (#16941)
This commit is contained in:
Michael
697
.github/workflows/nightly-test-amd.yml
vendored
697
.github/workflows/nightly-test-amd.yml
vendored
@@ -17,22 +17,24 @@ on:
|
||||
default: 'all'
|
||||
options:
|
||||
- 'all'
|
||||
- 'nightly-test-2-gpu'
|
||||
- 'nightly-test-2-gpu-vlm'
|
||||
- 'nightly-test-8-gpu-gpt-oss'
|
||||
- 'nightly-test-8-gpu-grok'
|
||||
- 'nightly-test-8-gpu-deepseek-r1'
|
||||
- 'nightly-perf-8-gpu-grok'
|
||||
- 'nightly-perf-8-gpu-deepseek-v3'
|
||||
- 'nightly-perf-8-gpu-deepseek-v31'
|
||||
# MI30x Unit Tests
|
||||
- 'nightly-test-1-gpu-unit'
|
||||
# MI30x Accuracy Tests (GSM8K / MMMU)
|
||||
- 'nightly-accuracy-2-gpu'
|
||||
- 'nightly-accuracy-2-gpu-vlm'
|
||||
- 'nightly-accuracy-8-gpu'
|
||||
- 'nightly-accuracy-8-gpu-deepseek-r1'
|
||||
# MI30x Accuracy + Performance Tests (combined)
|
||||
- 'nightly-8-gpu-grok1-int4'
|
||||
- 'nightly-8-gpu-grok2'
|
||||
- 'nightly-8-gpu-deepseek-v31'
|
||||
# MI35x jobs
|
||||
- 'nightly-test-2-gpu-mi35x'
|
||||
- 'nightly-test-2-gpu-vlm-mi35x'
|
||||
- 'nightly-test-8-gpu-mi35x-gpt-oss'
|
||||
- 'nightly-test-8-gpu-mi35x-grok'
|
||||
- 'nightly-test-8-gpu-mi35x-deepseek-r1'
|
||||
- 'nightly-perf-8-gpu-mi35x-grok'
|
||||
- 'nightly-perf-8-gpu-mi35x-deepseek-r1-mxfp4'
|
||||
- 'nightly-test-1-gpu-mi35x'
|
||||
- 'nightly-accuracy-8-gpu-mi35x'
|
||||
- 'nightly-accuracy-8-gpu-mi35x-deepseek-r1'
|
||||
- 'nightly-8-gpu-mi35x-grok1-int4'
|
||||
- 'nightly-8-gpu-mi35x-grok2'
|
||||
- 'nightly-8-gpu-mi35x-deepseek-r1-mxfp4'
|
||||
workflow_call:
|
||||
inputs:
|
||||
ref:
|
||||
@@ -51,9 +53,40 @@ concurrency:
|
||||
cancel-in-progress: ${{ github.event_name != 'workflow_call' }}
|
||||
|
||||
jobs:
|
||||
# 2-GPU tests (TP=2)
|
||||
nightly-test-2-gpu:
|
||||
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-2-gpu')
|
||||
# ============================================== MI30x Unit Tests ==============================================
|
||||
# 1-GPU Unit Tests - LoRA, debug utils, scheduler, etc. (MI30x only)
|
||||
nightly-test-1-gpu-unit:
|
||||
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-1-gpu-unit')
|
||||
runs-on: linux-mi325-gpu-1
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ inputs.ref || github.ref }}
|
||||
|
||||
- name: Setup docker
|
||||
run: |
|
||||
touch github_summary.md
|
||||
bash scripts/ci/amd_ci_start_container.sh
|
||||
env:
|
||||
GITHUB_WORKSPACE: ${{ github.workspace }}
|
||||
|
||||
- name: Install dependencies
|
||||
run: bash scripts/ci/amd_ci_install_dependency.sh
|
||||
|
||||
- name: Nightly Unit Test (1-GPU)
|
||||
timeout-minutes: 60
|
||||
run: |
|
||||
bash scripts/ci/amd_ci_exec.sh -w /sglang-checkout/test \
|
||||
-e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
|
||||
python3 run_suite.py --hw amd --suite nightly-amd-1-gpu --nightly --timeout-per-file 600 --continue-on-error || TEST_EXIT_CODE=$?
|
||||
echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
|
||||
exit ${TEST_EXIT_CODE:-0}
|
||||
|
||||
# ============================================== MI30x Accuracy Tests ==============================================
|
||||
# 2-GPU Accuracy Tests - GSM8K eval (MI30x only)
|
||||
nightly-accuracy-2-gpu:
|
||||
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-accuracy-2-gpu')
|
||||
runs-on: linux-mi325-gpu-2
|
||||
steps:
|
||||
- name: Checkout code
|
||||
@@ -79,9 +112,9 @@ jobs:
|
||||
echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
|
||||
exit ${TEST_EXIT_CODE:-0}
|
||||
|
||||
# 2-GPU VLM tests - Vision-Language Models MMMU evaluation
|
||||
nightly-test-2-gpu-vlm:
|
||||
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-2-gpu-vlm')
|
||||
# 2-GPU VLM Accuracy Tests - Vision-Language Models MMMU evaluation
|
||||
nightly-accuracy-2-gpu-vlm:
|
||||
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-accuracy-2-gpu-vlm')
|
||||
runs-on: linux-mi325-gpu-2
|
||||
steps:
|
||||
- name: Checkout code
|
||||
@@ -99,18 +132,18 @@ jobs:
|
||||
- name: Install dependencies
|
||||
run: bash scripts/ci/amd_ci_install_dependency.sh
|
||||
|
||||
- name: Nightly Test (2-GPU VLM MMMU)
|
||||
- name: Nightly Accuracy Test (2-GPU VLM MMMU)
|
||||
timeout-minutes: 180
|
||||
run: |
|
||||
bash scripts/ci/amd_ci_exec.sh -w /sglang-checkout/test \
|
||||
-e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
|
||||
python3 run_suite.py --hw amd --suite nightly-amd-vlm --nightly --timeout-per-file 7200 || TEST_EXIT_CODE=$?
|
||||
python3 run_suite.py --hw amd --suite nightly-amd-accuracy-2-gpu-vlm --nightly --timeout-per-file 7200 || TEST_EXIT_CODE=$?
|
||||
echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
|
||||
exit ${TEST_EXIT_CODE:-0}
|
||||
|
||||
# 8-GPU tests (TP=8) - GPT-OSS models
|
||||
nightly-test-8-gpu-gpt-oss:
|
||||
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-8-gpu-gpt-oss')
|
||||
# 8-GPU Accuracy Tests - GPT-OSS, Grok1-FP8 (accuracy only)
|
||||
nightly-accuracy-8-gpu:
|
||||
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-accuracy-8-gpu')
|
||||
runs-on: linux-mi325-gpu-8
|
||||
steps:
|
||||
- name: Checkout code
|
||||
@@ -128,103 +161,28 @@ jobs:
|
||||
- name: Install dependencies
|
||||
run: bash scripts/ci/amd_ci_install_dependency.sh
|
||||
|
||||
- name: Nightly Test (8-GPU GPT-OSS)
|
||||
- name: Accuracy Test (8-GPU GPT-OSS)
|
||||
timeout-minutes: 180
|
||||
run: |
|
||||
bash scripts/ci/amd_ci_exec.sh -w /sglang-checkout/test \
|
||||
-e AMD_TEST_MODEL_GROUP=gpt-oss \
|
||||
-e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
|
||||
python3 run_suite.py --hw amd --suite nightly-amd-8-gpu --nightly --timeout-per-file 7200 || TEST_EXIT_CODE=$?
|
||||
python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-gpt-oss --nightly --timeout-per-file 7200 || TEST_EXIT_CODE=$?
|
||||
echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
|
||||
exit ${TEST_EXIT_CODE:-0}
|
||||
|
||||
# 8-GPU tests (TP=8) - GROK models (GROK1-FP8, GROK1-IN4, GROK2.5)
|
||||
nightly-test-8-gpu-grok:
|
||||
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-8-gpu-grok')
|
||||
runs-on: linux-mi325-gpu-8
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ inputs.ref || github.ref }}
|
||||
|
||||
- name: Setup docker
|
||||
run: |
|
||||
touch github_summary.md
|
||||
bash scripts/ci/amd_ci_start_container.sh
|
||||
env:
|
||||
GITHUB_WORKSPACE: ${{ github.workspace }}
|
||||
|
||||
- name: Install dependencies
|
||||
run: bash scripts/ci/amd_ci_install_dependency.sh
|
||||
|
||||
- name: Nightly Test (8-GPU GROK)
|
||||
run: |
|
||||
bash scripts/ci/amd_ci_exec.sh -w /sglang-checkout/test \
|
||||
-e AMD_TEST_MODEL_GROUP=grok \
|
||||
-e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
|
||||
python3 run_suite.py --hw amd --suite nightly-amd-8-gpu --nightly --timeout-per-file 7200 || TEST_EXIT_CODE=$?
|
||||
echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
|
||||
exit ${TEST_EXIT_CODE:-0}
|
||||
|
||||
# 8-GPU tests (TP=8) - DeepSeek-R1 all variants (basic, MTP, DP, TC)
|
||||
nightly-test-8-gpu-deepseek-r1:
|
||||
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-8-gpu-deepseek-r1')
|
||||
runs-on: linux-mi325-gpu-8
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ inputs.ref || github.ref }}
|
||||
|
||||
- name: Setup docker
|
||||
run: |
|
||||
touch github_summary.md
|
||||
bash scripts/ci/amd_ci_start_container.sh
|
||||
env:
|
||||
GITHUB_WORKSPACE: ${{ github.workspace }}
|
||||
|
||||
- name: Install dependencies
|
||||
run: bash scripts/ci/amd_ci_install_dependency.sh
|
||||
|
||||
- name: Nightly Test (8-GPU DeepSeek-R1 all variants)
|
||||
run: |
|
||||
bash scripts/ci/amd_ci_exec.sh -w /sglang-checkout/test \
|
||||
-e AMD_TEST_MODEL_GROUP=deepseek-r1-all \
|
||||
-e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
|
||||
python3 run_suite.py --hw amd --suite nightly-amd-8-gpu --nightly --timeout-per-file 7200 || TEST_EXIT_CODE=$?
|
||||
echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
|
||||
exit ${TEST_EXIT_CODE:-0}
|
||||
|
||||
# 8-GPU Performance Tests (TP=8) - Grok (Grok-1 + Grok-2) performance benchmarks
|
||||
nightly-perf-8-gpu-grok:
|
||||
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-perf-8-gpu-grok')
|
||||
runs-on: linux-mi325-gpu-8
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ inputs.ref || github.ref }}
|
||||
|
||||
- name: Setup docker
|
||||
run: |
|
||||
touch github_summary.md
|
||||
bash scripts/ci/amd_ci_start_container.sh
|
||||
env:
|
||||
GITHUB_WORKSPACE: ${{ github.workspace }}
|
||||
|
||||
- name: Install dependencies
|
||||
run: bash scripts/ci/amd_ci_install_dependency.sh
|
||||
|
||||
- name: Nightly Perf Test (8-GPU Grok-1 + Grok-2)
|
||||
- name: Accuracy Test (8-GPU Grok1-FP8)
|
||||
timeout-minutes: 60
|
||||
run: |
|
||||
bash scripts/ci/amd_ci_exec.sh -w /sglang-checkout/test -e RCCL_MSCCL_ENABLE=0 -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" python3 registered/amd/test_grok_perf.py || TEST_EXIT_CODE=$?
|
||||
bash scripts/ci/amd_ci_exec.sh -w /sglang-checkout/test \
|
||||
-e RCCL_MSCCL_ENABLE=0 \
|
||||
-e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
|
||||
python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-grok1-fp8 --nightly --timeout-per-file 3600 || TEST_EXIT_CODE=$?
|
||||
echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
|
||||
exit ${TEST_EXIT_CODE:-0}
|
||||
|
||||
# 8-GPU Performance Tests (TP=8) - DeepSeek-V3 performance benchmarks
|
||||
nightly-perf-8-gpu-deepseek-v3:
|
||||
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-perf-8-gpu-deepseek-v3')
|
||||
# 8-GPU DeepSeek-R1 Accuracy Test (separate job due to long loading time)
|
||||
nightly-accuracy-8-gpu-deepseek-r1:
|
||||
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-accuracy-8-gpu-deepseek-r1')
|
||||
runs-on: linux-mi325-gpu-8
|
||||
steps:
|
||||
- name: Checkout code
|
||||
@@ -242,16 +200,19 @@ jobs:
|
||||
- name: Install dependencies
|
||||
run: bash scripts/ci/amd_ci_install_dependency.sh
|
||||
|
||||
- name: Nightly Perf Test (8-GPU DeepSeek-V3)
|
||||
timeout-minutes: 300
|
||||
- name: Accuracy Test (8-GPU DeepSeek-R1)
|
||||
timeout-minutes: 240
|
||||
run: |
|
||||
bash scripts/ci/amd_ci_exec.sh -w /sglang-checkout/test -e SGLANG_USE_ROCM700A=1 -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" python3 registered/amd/test_deepseek_v3_perf.py || TEST_EXIT_CODE=$?
|
||||
bash scripts/ci/amd_ci_exec.sh -w /sglang-checkout/test \
|
||||
-e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
|
||||
python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-deepseek-r1 --nightly --timeout-per-file 7200 || TEST_EXIT_CODE=$?
|
||||
echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
|
||||
exit ${TEST_EXIT_CODE:-0}
|
||||
|
||||
# 8-GPU Performance Tests (TP=8) - DeepSeek-V3.1 performance benchmarks
|
||||
nightly-perf-8-gpu-deepseek-v31:
|
||||
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-perf-8-gpu-deepseek-v31')
|
||||
# ============================================== MI30x Combined Accuracy + Performance Tests ==============================================
|
||||
# 8-GPU Grok1-INT4 (Accuracy + Performance combined)
|
||||
nightly-8-gpu-grok1-int4:
|
||||
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-8-gpu-grok1-int4')
|
||||
runs-on: linux-mi325-gpu-8
|
||||
steps:
|
||||
- name: Checkout code
|
||||
@@ -269,18 +230,120 @@ jobs:
|
||||
- name: Install dependencies
|
||||
run: bash scripts/ci/amd_ci_install_dependency.sh
|
||||
|
||||
- name: Nightly Perf Test (8-GPU DeepSeek-V3.1)
|
||||
timeout-minutes: 300
|
||||
- name: Accuracy Test (8-GPU Grok1-INT4)
|
||||
timeout-minutes: 60
|
||||
run: |
|
||||
bash scripts/ci/amd_ci_exec.sh -w /sglang-checkout/test -e SGLANG_USE_ROCM700A=1 -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" python3 registered/amd/test_deepseek_v31_perf.py || TEST_EXIT_CODE=$?
|
||||
> github_summary.md # Clear summary file
|
||||
bash scripts/ci/amd_ci_exec.sh -w /sglang-checkout/test \
|
||||
-e RCCL_MSCCL_ENABLE=0 \
|
||||
-e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
|
||||
python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-grok1-int4 --nightly --timeout-per-file 3600 || TEST_EXIT_CODE=$?
|
||||
echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
|
||||
exit ${TEST_EXIT_CODE:-0}
|
||||
|
||||
- name: Performance Test (8-GPU Grok1-INT4)
|
||||
timeout-minutes: 60
|
||||
continue-on-error: true # Perf test failure doesn't fail the job if accuracy passed
|
||||
run: |
|
||||
> github_summary.md # Clear summary file
|
||||
bash scripts/ci/amd_ci_exec.sh -w /sglang-checkout/test \
|
||||
-e RCCL_MSCCL_ENABLE=0 \
|
||||
-e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
|
||||
python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-grok1-int4 --nightly --timeout-per-file 3600 || TEST_EXIT_CODE=$?
|
||||
echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
|
||||
exit ${TEST_EXIT_CODE:-0}
|
||||
|
||||
# 8-GPU Grok2 (Accuracy + Performance combined)
|
||||
nightly-8-gpu-grok2:
|
||||
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-8-gpu-grok2')
|
||||
runs-on: linux-mi325-gpu-8
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ inputs.ref || github.ref }}
|
||||
|
||||
- name: Setup docker
|
||||
run: |
|
||||
touch github_summary.md
|
||||
bash scripts/ci/amd_ci_start_container.sh
|
||||
env:
|
||||
GITHUB_WORKSPACE: ${{ github.workspace }}
|
||||
|
||||
- name: Install dependencies
|
||||
run: bash scripts/ci/amd_ci_install_dependency.sh
|
||||
|
||||
- name: Accuracy Test (8-GPU Grok2)
|
||||
timeout-minutes: 60
|
||||
run: |
|
||||
> github_summary.md # Clear summary file
|
||||
bash scripts/ci/amd_ci_exec.sh -w /sglang-checkout/test \
|
||||
-e RCCL_MSCCL_ENABLE=0 \
|
||||
-e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
|
||||
python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-grok2 --nightly --timeout-per-file 3600 || TEST_EXIT_CODE=$?
|
||||
echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
|
||||
exit ${TEST_EXIT_CODE:-0}
|
||||
|
||||
- name: Performance Test (8-GPU Grok2)
|
||||
timeout-minutes: 60
|
||||
continue-on-error: true # Perf test failure doesn't fail the job if accuracy passed
|
||||
run: |
|
||||
> github_summary.md # Clear summary file
|
||||
bash scripts/ci/amd_ci_exec.sh -w /sglang-checkout/test \
|
||||
-e RCCL_MSCCL_ENABLE=0 \
|
||||
-e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
|
||||
python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-grok2 --nightly --timeout-per-file 3600 || TEST_EXIT_CODE=$?
|
||||
echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
|
||||
exit ${TEST_EXIT_CODE:-0}
|
||||
|
||||
# 8-GPU DeepSeek-V3.1 (Accuracy + Performance combined)
|
||||
nightly-8-gpu-deepseek-v31:
|
||||
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-8-gpu-deepseek-v31')
|
||||
runs-on: linux-mi325-gpu-8
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ inputs.ref || github.ref }}
|
||||
|
||||
- name: Setup docker
|
||||
run: |
|
||||
touch github_summary.md
|
||||
bash scripts/ci/amd_ci_start_container.sh
|
||||
env:
|
||||
GITHUB_WORKSPACE: ${{ github.workspace }}
|
||||
|
||||
- name: Install dependencies
|
||||
run: bash scripts/ci/amd_ci_install_dependency.sh
|
||||
|
||||
- name: Accuracy Test (8-GPU DeepSeek-V3.1)
|
||||
timeout-minutes: 120
|
||||
run: |
|
||||
> github_summary.md # Clear summary file
|
||||
bash scripts/ci/amd_ci_exec.sh -w /sglang-checkout/test \
|
||||
-e SGLANG_USE_AITER=1 \
|
||||
-e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
|
||||
python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-deepseek-v31 --nightly --timeout-per-file 3600 || TEST_EXIT_CODE=$?
|
||||
echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
|
||||
exit ${TEST_EXIT_CODE:-0}
|
||||
|
||||
- name: Performance Test (8-GPU DeepSeek-V3.1)
|
||||
timeout-minutes: 300
|
||||
continue-on-error: true # Perf test failure doesn't fail the job if accuracy passed
|
||||
run: |
|
||||
> github_summary.md # Clear summary file
|
||||
bash scripts/ci/amd_ci_exec.sh -w /sglang-checkout/test \
|
||||
-e SGLANG_USE_ROCM700A=1 \
|
||||
-e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
|
||||
python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-deepseek-v31 --nightly --timeout-per-file 18000 || TEST_EXIT_CODE=$?
|
||||
echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
|
||||
exit ${TEST_EXIT_CODE:-0}
|
||||
|
||||
# ============================================== MI35x Tests ==============================================
|
||||
# MI35x 2-GPU tests (TP=2) - Reuses nightly-amd suite
|
||||
nightly-test-2-gpu-mi35x:
|
||||
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-2-gpu-mi35x')
|
||||
runs-on: linux-mi35x-gpu-2
|
||||
# MI35x 1-GPU tests - platform-agnostic tests that may work on CDNA4 (gfx950)
|
||||
nightly-test-1-gpu-mi35x:
|
||||
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-1-gpu-mi35x')
|
||||
runs-on: linux-mi35x-gpu-1
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
@@ -300,179 +363,18 @@ jobs:
|
||||
# Install tabulate for run_suite.py (missing in MI35x container)
|
||||
bash scripts/ci/amd_ci_exec.sh pip install tabulate
|
||||
|
||||
- name: Nightly Test (2-GPU)
|
||||
run: |
|
||||
bash scripts/ci/amd_ci_exec.sh -w /sglang-checkout/test \
|
||||
-e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
|
||||
python3 run_suite.py --hw amd --suite nightly-amd --nightly --timeout-per-file 7200 || TEST_EXIT_CODE=$?
|
||||
echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
|
||||
exit ${TEST_EXIT_CODE:-0}
|
||||
|
||||
# MI35x 2-GPU VLM tests - Reuses nightly-amd-vlm suite
|
||||
nightly-test-2-gpu-vlm-mi35x:
|
||||
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-2-gpu-vlm-mi35x')
|
||||
runs-on: linux-mi35x-gpu-2
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ inputs.ref || github.ref }}
|
||||
|
||||
- name: Setup docker
|
||||
run: |
|
||||
touch github_summary.md
|
||||
bash scripts/ci/amd_ci_start_container.sh
|
||||
env:
|
||||
GITHUB_WORKSPACE: ${{ github.workspace }}
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
bash scripts/ci/amd_ci_install_dependency.sh
|
||||
# Install tabulate for run_suite.py (missing in MI35x container)
|
||||
bash scripts/ci/amd_ci_exec.sh pip install tabulate
|
||||
|
||||
- name: Nightly Test (2-GPU VLM MMMU)
|
||||
timeout-minutes: 180
|
||||
run: |
|
||||
bash scripts/ci/amd_ci_exec.sh -w /sglang-checkout/test \
|
||||
-e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
|
||||
python3 run_suite.py --hw amd --suite nightly-amd-vlm --nightly --timeout-per-file 7200 || TEST_EXIT_CODE=$?
|
||||
echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
|
||||
exit ${TEST_EXIT_CODE:-0}
|
||||
|
||||
# MI35x 8-GPU tests (TP=8) - GPT-OSS models (MI35x uses openai/* paths)
|
||||
nightly-test-8-gpu-mi35x-gpt-oss:
|
||||
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-8-gpu-mi35x-gpt-oss')
|
||||
runs-on: linux-mi35x-gpu-8
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ inputs.ref || github.ref }}
|
||||
|
||||
- name: Setup docker
|
||||
run: |
|
||||
touch github_summary.md
|
||||
bash scripts/ci/amd_ci_start_container.sh
|
||||
env:
|
||||
GITHUB_WORKSPACE: ${{ github.workspace }}
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
bash scripts/ci/amd_ci_install_dependency.sh
|
||||
# Install tabulate for run_suite.py (missing in MI35x container)
|
||||
bash scripts/ci/amd_ci_exec.sh pip install tabulate
|
||||
|
||||
- name: Nightly Test MI35x (8-GPU GPT-OSS)
|
||||
timeout-minutes: 180
|
||||
run: |
|
||||
bash scripts/ci/amd_ci_exec.sh -w /sglang-checkout/test \
|
||||
-e AMD_TEST_MODEL_GROUP=gpt-oss \
|
||||
-e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
|
||||
python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x --nightly --timeout-per-file 7200 || TEST_EXIT_CODE=$?
|
||||
echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
|
||||
exit ${TEST_EXIT_CODE:-0}
|
||||
|
||||
# MI35x 8-GPU tests (TP=8) - GROK models
|
||||
nightly-test-8-gpu-mi35x-grok:
|
||||
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-8-gpu-mi35x-grok')
|
||||
runs-on: linux-mi35x-gpu-8
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ inputs.ref || github.ref }}
|
||||
|
||||
- name: Setup docker
|
||||
run: |
|
||||
touch github_summary.md
|
||||
bash scripts/ci/amd_ci_start_container.sh
|
||||
env:
|
||||
GITHUB_WORKSPACE: ${{ github.workspace }}
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
bash scripts/ci/amd_ci_install_dependency.sh
|
||||
# Install tabulate for run_suite.py (missing in MI35x container)
|
||||
bash scripts/ci/amd_ci_exec.sh pip install tabulate
|
||||
|
||||
- name: Nightly Test MI35x (8-GPU GROK)
|
||||
run: |
|
||||
bash scripts/ci/amd_ci_exec.sh -w /sglang-checkout/test \
|
||||
-e AMD_TEST_MODEL_GROUP=grok \
|
||||
-e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
|
||||
python3 run_suite.py --hw amd --suite nightly-amd-8-gpu --nightly --timeout-per-file 7200 || TEST_EXIT_CODE=$?
|
||||
echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
|
||||
exit ${TEST_EXIT_CODE:-0}
|
||||
|
||||
# MI35x 8-GPU tests (TP=8) - DeepSeek-R1-0528 basic + MTP only
|
||||
# Same model as MI300X for consistency; MXFP4 only used for perf tests
|
||||
# Note: DP/TC variants disabled for MI35x due to initialization timeouts
|
||||
nightly-test-8-gpu-mi35x-deepseek-r1:
|
||||
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-8-gpu-mi35x-deepseek-r1')
|
||||
runs-on: linux-mi35x-gpu-8
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ inputs.ref || github.ref }}
|
||||
|
||||
- name: Setup docker
|
||||
run: |
|
||||
touch github_summary.md
|
||||
bash scripts/ci/amd_ci_start_container.sh
|
||||
env:
|
||||
GITHUB_WORKSPACE: ${{ github.workspace }}
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
bash scripts/ci/amd_ci_install_dependency.sh
|
||||
# Install tabulate for run_suite.py (missing in MI35x container)
|
||||
bash scripts/ci/amd_ci_exec.sh pip install tabulate
|
||||
|
||||
- name: Nightly Test MI35x (8-GPU DeepSeek-R1-0528 basic + MTP)
|
||||
timeout-minutes: 180
|
||||
run: |
|
||||
bash scripts/ci/amd_ci_exec.sh -w /sglang-checkout/test \
|
||||
-e AMD_TEST_MODEL_GROUP=deepseek-r1 \
|
||||
-e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
|
||||
python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x --nightly --timeout-per-file 7200 || TEST_EXIT_CODE=$?
|
||||
echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
|
||||
exit ${TEST_EXIT_CODE:-0}
|
||||
|
||||
# MI35x 8-GPU Performance Tests (TP=8) - Grok performance benchmarks
|
||||
nightly-perf-8-gpu-mi35x-grok:
|
||||
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-perf-8-gpu-mi35x-grok')
|
||||
runs-on: linux-mi35x-gpu-8
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ inputs.ref || github.ref }}
|
||||
|
||||
- name: Setup docker
|
||||
run: |
|
||||
touch github_summary.md
|
||||
bash scripts/ci/amd_ci_start_container.sh
|
||||
env:
|
||||
GITHUB_WORKSPACE: ${{ github.workspace }}
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
bash scripts/ci/amd_ci_install_dependency.sh
|
||||
# Install tabulate for run_suite.py (missing in MI35x container)
|
||||
bash scripts/ci/amd_ci_exec.sh pip install tabulate
|
||||
|
||||
- name: Nightly Perf Test MI35x (8-GPU Grok)
|
||||
- name: Nightly Test MI35x (1-GPU)
|
||||
timeout-minutes: 60
|
||||
run: |
|
||||
bash scripts/ci/amd_ci_exec.sh -w /sglang-checkout/test -e RCCL_MSCCL_ENABLE=0 -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" python3 registered/amd/test_grok_perf.py || TEST_EXIT_CODE=$?
|
||||
bash scripts/ci/amd_ci_exec.sh -w /sglang-checkout/test \
|
||||
-e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
|
||||
python3 run_suite.py --hw amd --suite nightly-amd-1-gpu-mi35x --nightly --timeout-per-file 600 --continue-on-error || TEST_EXIT_CODE=$?
|
||||
echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
|
||||
exit ${TEST_EXIT_CODE:-0}
|
||||
|
||||
# MI35x 8-GPU Performance Tests (TP=8) - DeepSeek-R1-MXFP4 performance benchmarks
|
||||
nightly-perf-8-gpu-mi35x-deepseek-r1-mxfp4:
|
||||
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-perf-8-gpu-mi35x-deepseek-r1-mxfp4')
|
||||
# MI35x 8-GPU Accuracy Tests - GPT-OSS (accuracy only)
|
||||
nightly-accuracy-8-gpu-mi35x:
|
||||
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-accuracy-8-gpu-mi35x')
|
||||
runs-on: linux-mi35x-gpu-8
|
||||
steps:
|
||||
- name: Checkout code
|
||||
@@ -493,33 +395,204 @@ jobs:
|
||||
# Install tabulate for run_suite.py (missing in MI35x container)
|
||||
bash scripts/ci/amd_ci_exec.sh pip install tabulate
|
||||
|
||||
- name: Nightly Perf Test MI35x (8-GPU DeepSeek-R1-MXFP4)
|
||||
timeout-minutes: 300
|
||||
- name: Accuracy Test MI35x (8-GPU GPT-OSS)
|
||||
timeout-minutes: 180
|
||||
run: |
|
||||
bash scripts/ci/amd_ci_exec.sh -w /sglang-checkout/test -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" python3 registered/amd/test_deepseek_r1_mxfp4_perf.py || TEST_EXIT_CODE=$?
|
||||
bash scripts/ci/amd_ci_exec.sh -w /sglang-checkout/test \
|
||||
-e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
|
||||
python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x --nightly --timeout-per-file 7200 || TEST_EXIT_CODE=$?
|
||||
echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
|
||||
exit ${TEST_EXIT_CODE:-0}
|
||||
|
||||
# MI35x 8-GPU DeepSeek-R1-0528 Accuracy Test (separate job due to long loading time)
|
||||
nightly-accuracy-8-gpu-mi35x-deepseek-r1:
|
||||
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-accuracy-8-gpu-mi35x-deepseek-r1')
|
||||
runs-on: linux-mi35x-gpu-8
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ inputs.ref || github.ref }}
|
||||
|
||||
- name: Setup docker
|
||||
run: |
|
||||
touch github_summary.md
|
||||
bash scripts/ci/amd_ci_start_container.sh
|
||||
env:
|
||||
GITHUB_WORKSPACE: ${{ github.workspace }}
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
bash scripts/ci/amd_ci_install_dependency.sh
|
||||
# Install tabulate for run_suite.py (missing in MI35x container)
|
||||
bash scripts/ci/amd_ci_exec.sh pip install tabulate
|
||||
|
||||
- name: Accuracy Test MI35x (8-GPU DeepSeek-R1-0528)
|
||||
timeout-minutes: 240
|
||||
run: |
|
||||
bash scripts/ci/amd_ci_exec.sh -w /sglang-checkout/test \
|
||||
-e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
|
||||
python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-mi35x-deepseek-r1 --nightly --timeout-per-file 7200 || TEST_EXIT_CODE=$?
|
||||
echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
|
||||
exit ${TEST_EXIT_CODE:-0}
|
||||
|
||||
# MI35x 8-GPU Grok1-INT4 (Accuracy + Performance combined)
|
||||
nightly-8-gpu-mi35x-grok1-int4:
|
||||
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-8-gpu-mi35x-grok1-int4')
|
||||
runs-on: linux-mi35x-gpu-8
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ inputs.ref || github.ref }}
|
||||
|
||||
- name: Setup docker
|
||||
run: |
|
||||
touch github_summary.md
|
||||
bash scripts/ci/amd_ci_start_container.sh
|
||||
env:
|
||||
GITHUB_WORKSPACE: ${{ github.workspace }}
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
bash scripts/ci/amd_ci_install_dependency.sh
|
||||
# Install tabulate for run_suite.py (missing in MI35x container)
|
||||
bash scripts/ci/amd_ci_exec.sh pip install tabulate
|
||||
|
||||
- name: Accuracy Test MI35x (8-GPU Grok1-INT4)
|
||||
timeout-minutes: 60
|
||||
run: |
|
||||
> github_summary.md # Clear summary file
|
||||
bash scripts/ci/amd_ci_exec.sh -w /sglang-checkout/test \
|
||||
-e RCCL_MSCCL_ENABLE=0 \
|
||||
-e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
|
||||
python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-mi35x-grok1-int4 --nightly --timeout-per-file 3600 || TEST_EXIT_CODE=$?
|
||||
echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
|
||||
exit ${TEST_EXIT_CODE:-0}
|
||||
|
||||
- name: Performance Test MI35x (8-GPU Grok1-INT4)
|
||||
timeout-minutes: 60
|
||||
continue-on-error: true # Perf test failure doesn't fail the job if accuracy passed
|
||||
run: |
|
||||
> github_summary.md # Clear summary file
|
||||
bash scripts/ci/amd_ci_exec.sh -w /sglang-checkout/test \
|
||||
-e RCCL_MSCCL_ENABLE=0 \
|
||||
-e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
|
||||
python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-mi35x-grok1-int4 --nightly --timeout-per-file 3600 || TEST_EXIT_CODE=$?
|
||||
echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
|
||||
exit ${TEST_EXIT_CODE:-0}
|
||||
|
||||
# MI35x 8-GPU Grok2 (Accuracy + Performance combined)
|
||||
nightly-8-gpu-mi35x-grok2:
|
||||
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-8-gpu-mi35x-grok2')
|
||||
runs-on: linux-mi35x-gpu-8
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ inputs.ref || github.ref }}
|
||||
|
||||
- name: Setup docker
|
||||
run: |
|
||||
touch github_summary.md
|
||||
bash scripts/ci/amd_ci_start_container.sh
|
||||
env:
|
||||
GITHUB_WORKSPACE: ${{ github.workspace }}
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
bash scripts/ci/amd_ci_install_dependency.sh
|
||||
# Install tabulate for run_suite.py (missing in MI35x container)
|
||||
bash scripts/ci/amd_ci_exec.sh pip install tabulate
|
||||
|
||||
- name: Accuracy Test MI35x (8-GPU Grok2)
|
||||
timeout-minutes: 60
|
||||
run: |
|
||||
> github_summary.md # Clear summary file
|
||||
bash scripts/ci/amd_ci_exec.sh -w /sglang-checkout/test \
|
||||
-e RCCL_MSCCL_ENABLE=0 \
|
||||
-e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
|
||||
python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-mi35x-grok2 --nightly --timeout-per-file 3600 || TEST_EXIT_CODE=$?
|
||||
echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
|
||||
exit ${TEST_EXIT_CODE:-0}
|
||||
|
||||
- name: Performance Test MI35x (8-GPU Grok2)
|
||||
timeout-minutes: 60
|
||||
continue-on-error: true # Perf test failure doesn't fail the job if accuracy passed
|
||||
run: |
|
||||
> github_summary.md # Clear summary file
|
||||
bash scripts/ci/amd_ci_exec.sh -w /sglang-checkout/test \
|
||||
-e RCCL_MSCCL_ENABLE=0 \
|
||||
-e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
|
||||
python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-mi35x-grok2 --nightly --timeout-per-file 3600 || TEST_EXIT_CODE=$?
|
||||
echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
|
||||
exit ${TEST_EXIT_CODE:-0}
|
||||
|
||||
# MI35x 8-GPU DeepSeek-R1-MXFP4 (Accuracy + Performance combined)
|
||||
nightly-8-gpu-mi35x-deepseek-r1-mxfp4:
|
||||
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-8-gpu-mi35x-deepseek-r1-mxfp4')
|
||||
runs-on: linux-mi35x-gpu-8
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ inputs.ref || github.ref }}
|
||||
|
||||
- name: Setup docker
|
||||
run: |
|
||||
touch github_summary.md
|
||||
bash scripts/ci/amd_ci_start_container.sh
|
||||
env:
|
||||
GITHUB_WORKSPACE: ${{ github.workspace }}
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
bash scripts/ci/amd_ci_install_dependency.sh
|
||||
# Install tabulate for run_suite.py (missing in MI35x container)
|
||||
bash scripts/ci/amd_ci_exec.sh pip install tabulate
|
||||
|
||||
- name: Accuracy Test MI35x (8-GPU DeepSeek-R1-MXFP4)
|
||||
timeout-minutes: 180
|
||||
run: |
|
||||
> github_summary.md # Clear summary file
|
||||
bash scripts/ci/amd_ci_exec.sh -w /sglang-checkout/test \
|
||||
-e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
|
||||
python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x-deepseek-r1-mxfp4 --nightly --timeout-per-file 7200 || TEST_EXIT_CODE=$?
|
||||
echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
|
||||
exit ${TEST_EXIT_CODE:-0}
|
||||
|
||||
- name: Performance Test MI35x (8-GPU DeepSeek-R1-MXFP4)
|
||||
timeout-minutes: 300
|
||||
continue-on-error: true # Perf test failure doesn't fail the job if accuracy passed
|
||||
run: |
|
||||
> github_summary.md # Clear summary file
|
||||
bash scripts/ci/amd_ci_exec.sh -w /sglang-checkout/test \
|
||||
-e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
|
||||
python3 registered/amd/perf/mi35x/test_deepseek_r1_mxfp4_perf_mi35x.py || TEST_EXIT_CODE=$?
|
||||
echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
|
||||
exit ${TEST_EXIT_CODE:-0}
|
||||
|
||||
check-all-jobs:
|
||||
if: always() && (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch')
|
||||
needs:
|
||||
# MI325 jobs
|
||||
- nightly-test-2-gpu
|
||||
- nightly-test-2-gpu-vlm
|
||||
- nightly-test-8-gpu-gpt-oss
|
||||
- nightly-test-8-gpu-grok
|
||||
- nightly-test-8-gpu-deepseek-r1
|
||||
- nightly-perf-8-gpu-grok
|
||||
- nightly-perf-8-gpu-deepseek-v3
|
||||
- nightly-perf-8-gpu-deepseek-v31
|
||||
# MI30x Unit Tests
|
||||
- nightly-test-1-gpu-unit
|
||||
# MI30x Accuracy Tests
|
||||
- nightly-accuracy-2-gpu
|
||||
- nightly-accuracy-2-gpu-vlm
|
||||
- nightly-accuracy-8-gpu
|
||||
- nightly-accuracy-8-gpu-deepseek-r1
|
||||
# MI30x Combined Accuracy + Performance Tests
|
||||
- nightly-8-gpu-grok1-int4
|
||||
- nightly-8-gpu-grok2
|
||||
- nightly-8-gpu-deepseek-v31
|
||||
# MI35x jobs
|
||||
- nightly-test-2-gpu-mi35x
|
||||
- nightly-test-2-gpu-vlm-mi35x
|
||||
- nightly-test-8-gpu-mi35x-gpt-oss
|
||||
- nightly-test-8-gpu-mi35x-grok
|
||||
- nightly-test-8-gpu-mi35x-deepseek-r1
|
||||
- nightly-perf-8-gpu-mi35x-grok
|
||||
- nightly-perf-8-gpu-mi35x-deepseek-r1-mxfp4
|
||||
- nightly-test-1-gpu-mi35x
|
||||
- nightly-accuracy-8-gpu-mi35x
|
||||
- nightly-accuracy-8-gpu-mi35x-deepseek-r1
|
||||
- nightly-8-gpu-mi35x-grok1-int4
|
||||
- nightly-8-gpu-mi35x-grok2
|
||||
- nightly-8-gpu-mi35x-deepseek-r1-mxfp4
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Check if any job failed
|
||||
|
||||
@@ -0,0 +1,250 @@
|
||||
"""MI35x DeepSeek-R1 GSM8K Completion Evaluation Test (8-GPU)
|
||||
|
||||
Tests DeepSeek-R1-0528 with basic configuration using few-shot completion
|
||||
benchmark on MI35x.
|
||||
|
||||
Registry: nightly-amd-accuracy-8-gpu-mi35x-deepseek-r1 suite
|
||||
"""
|
||||
|
||||
import ast
|
||||
import os
|
||||
|
||||
# Set HF cache for MI35x
|
||||
os.environ.setdefault("HF_HOME", "/data2/models/huggingface")
|
||||
os.environ.setdefault("HF_HUB_CACHE", "/data2/models/huggingface/hub")
|
||||
|
||||
import re
|
||||
import time
|
||||
import unittest
|
||||
from dataclasses import dataclass
|
||||
from typing import List, Optional, Tuple
|
||||
|
||||
import numpy as np
|
||||
|
||||
from sglang.srt.utils import kill_process_tree
|
||||
from sglang.test.ci.ci_register import register_amd_ci
|
||||
from sglang.test.test_utils import (
|
||||
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||
DEFAULT_URL_FOR_TEST,
|
||||
is_in_ci,
|
||||
popen_launch_server,
|
||||
write_github_step_summary,
|
||||
)
|
||||
from sglang.utils import download_and_cache_file, read_jsonl
|
||||
|
||||
# Register for AMD CI - MI35x DeepSeek-R1-0528 accuracy test (~30 min for basic only)
|
||||
register_amd_ci(
|
||||
est_time=1800,
|
||||
suite="nightly-amd-accuracy-8-gpu-mi35x-deepseek-r1",
|
||||
nightly=True,
|
||||
)
|
||||
|
||||
INVALID = -9999999
|
||||
|
||||
|
||||
@dataclass
|
||||
class ModelConfig:
|
||||
"""Configuration for a model to test."""
|
||||
|
||||
model_path: str
|
||||
tp_size: int = 8
|
||||
accuracy_threshold: float = 0.50
|
||||
other_args: Optional[List[str]] = None
|
||||
env_vars: Optional[dict] = None
|
||||
timeout: Optional[int] = None
|
||||
variant: Optional[str] = None
|
||||
|
||||
def __post_init__(self):
|
||||
if self.other_args is None:
|
||||
self.other_args = []
|
||||
if self.env_vars is None:
|
||||
self.env_vars = {}
|
||||
|
||||
def get_display_name(self) -> str:
|
||||
if self.variant:
|
||||
return f"{self.model_path} ({self.variant})"
|
||||
return self.model_path
|
||||
|
||||
|
||||
# DeepSeek-R1 models for MI35x - only basic variant for faster CI
|
||||
# MTP, DP, and TC variants removed to reduce test time from ~2h to ~30min
|
||||
MI35X_DEEPSEEK_R1_MODELS = [
|
||||
# DeepSeek-R1-0528 basic
|
||||
ModelConfig(
|
||||
model_path="deepseek-ai/DeepSeek-R1-0528",
|
||||
tp_size=8,
|
||||
accuracy_threshold=0.93,
|
||||
timeout=3600,
|
||||
variant="basic",
|
||||
other_args=[
|
||||
"--attention-backend",
|
||||
"aiter",
|
||||
"--chunked-prefill-size",
|
||||
"131072",
|
||||
"--disable-radix-cache",
|
||||
"--mem-fraction-static",
|
||||
"0.85",
|
||||
"--trust-remote-code",
|
||||
"--watchdog-timeout",
|
||||
"1200", # 20 minutes for weight loading
|
||||
],
|
||||
env_vars={"SGLANG_USE_AITER": "1"},
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def get_one_example(lines, i, include_answer):
|
||||
"""Format a single GSM8K example."""
|
||||
ret = "Question: " + lines[i]["question"] + "\nAnswer:"
|
||||
if include_answer:
|
||||
ret += " " + lines[i]["answer"]
|
||||
return ret
|
||||
|
||||
|
||||
def get_few_shot_examples(lines, k):
|
||||
"""Get k few-shot examples for prompting."""
|
||||
ret = ""
|
||||
for i in range(k):
|
||||
ret += get_one_example(lines, i, True) + "\n\n"
|
||||
return ret
|
||||
|
||||
|
||||
def get_answer_value(answer_str):
|
||||
"""Extract numerical answer from response."""
|
||||
answer_str = answer_str.replace(",", "")
|
||||
numbers = re.findall(r"\d+", answer_str)
|
||||
if len(numbers) < 1:
|
||||
return INVALID
|
||||
try:
|
||||
return ast.literal_eval(numbers[-1])
|
||||
except SyntaxError:
|
||||
return INVALID
|
||||
|
||||
|
||||
def run_gsm8k_benchmark(
|
||||
base_url: str,
|
||||
num_questions: int = 200,
|
||||
num_shots: int = 5,
|
||||
parallel: int = 64,
|
||||
) -> Tuple[float, float, float]:
|
||||
"""Run GSM8K few-shot completion benchmark."""
|
||||
import sglang as sgl
|
||||
from sglang.lang.backend.runtime_endpoint import RuntimeEndpoint
|
||||
|
||||
url = "https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/test.jsonl"
|
||||
data_path = download_and_cache_file(url)
|
||||
lines = list(read_jsonl(data_path))
|
||||
|
||||
few_shot_examples = get_few_shot_examples(lines, num_shots)
|
||||
|
||||
questions = []
|
||||
labels = []
|
||||
for i in range(len(lines[:num_questions])):
|
||||
questions.append(get_one_example(lines, i, False))
|
||||
labels.append(get_answer_value(lines[i]["answer"]))
|
||||
assert all(l != INVALID for l in labels)
|
||||
arguments = [{"question": q} for q in questions]
|
||||
|
||||
@sgl.function
|
||||
def few_shot_gsm8k(s, question):
|
||||
s += few_shot_examples + question
|
||||
s += sgl.gen(
|
||||
"answer", max_tokens=512, stop=["Question", "Assistant:", "<|separator|>"]
|
||||
)
|
||||
|
||||
backend = RuntimeEndpoint(base_url)
|
||||
sgl.set_default_backend(backend)
|
||||
|
||||
tic = time.perf_counter()
|
||||
states = few_shot_gsm8k.run_batch(
|
||||
arguments, temperature=0, num_threads=parallel, progress_bar=True
|
||||
)
|
||||
latency = time.perf_counter() - tic
|
||||
|
||||
preds = [get_answer_value(states[i]["answer"]) for i in range(len(states))]
|
||||
acc = np.mean(np.array(preds) == np.array(labels))
|
||||
invalid = np.mean(np.array(preds) == INVALID)
|
||||
|
||||
return float(acc), float(invalid), float(latency)
|
||||
|
||||
|
||||
class TestDeepSeekR1EvalMI35x(unittest.TestCase):
|
||||
"""DeepSeek-R1 GSM8K Completion Evaluation Test for AMD MI35x."""
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.models = MI35X_DEEPSEEK_R1_MODELS
|
||||
cls.base_url = DEFAULT_URL_FOR_TEST
|
||||
cls.num_questions = int(os.environ.get("GSM8K_NUM_QUESTIONS", "200"))
|
||||
|
||||
def test_deepseek_r1_accuracy(self):
|
||||
"""Test DeepSeek-R1 models with GSM8K completion benchmark."""
|
||||
all_results = []
|
||||
summary = "### DeepSeek-R1 Models (MI35x)\n\n"
|
||||
summary += "| Model | Variant | TP | Accuracy | Threshold | Status |\n"
|
||||
summary += "| ----- | ------- | -- | -------- | --------- | ------ |\n"
|
||||
|
||||
for config in self.models:
|
||||
display_name = config.get_display_name()
|
||||
with self.subTest(model=display_name):
|
||||
print(f"\n{'='*60}")
|
||||
print(f"Testing: {display_name}")
|
||||
print(f"{'='*60}")
|
||||
|
||||
env = os.environ.copy()
|
||||
for key, value in config.env_vars.items():
|
||||
env[key] = value
|
||||
|
||||
other_args = list(config.other_args)
|
||||
other_args.extend(["--tp", str(config.tp_size)])
|
||||
timeout = config.timeout or DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH
|
||||
|
||||
try:
|
||||
process = popen_launch_server(
|
||||
model=config.model_path,
|
||||
base_url=self.base_url,
|
||||
timeout=timeout,
|
||||
other_args=other_args,
|
||||
env=env,
|
||||
)
|
||||
|
||||
try:
|
||||
acc, invalid, latency = run_gsm8k_benchmark(
|
||||
self.base_url, num_questions=self.num_questions
|
||||
)
|
||||
passed = acc >= config.accuracy_threshold
|
||||
status = "✅ PASS" if passed else "❌ FAIL"
|
||||
|
||||
all_results.append(
|
||||
{
|
||||
"model": display_name,
|
||||
"accuracy": acc,
|
||||
"passed": passed,
|
||||
}
|
||||
)
|
||||
summary += f"| {config.model_path} | {config.variant or 'N/A'} | {config.tp_size} | {acc:.3f} | {config.accuracy_threshold} | {status} |\n"
|
||||
|
||||
finally:
|
||||
kill_process_tree(process.pid)
|
||||
|
||||
except Exception as e:
|
||||
summary += f"| {config.model_path} | {config.variant or 'N/A'} | {config.tp_size} | N/A | {config.accuracy_threshold} | ❌ ERROR |\n"
|
||||
all_results.append(
|
||||
{
|
||||
"model": display_name,
|
||||
"accuracy": None,
|
||||
"passed": False,
|
||||
"error": str(e),
|
||||
}
|
||||
)
|
||||
|
||||
if is_in_ci():
|
||||
write_github_step_summary(summary)
|
||||
|
||||
failed = [r for r in all_results if not r["passed"]]
|
||||
if failed:
|
||||
raise AssertionError(f"Failed models: {[r['model'] for r in failed]}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,300 @@
|
||||
"""MI35x DeepSeek-R1-MXFP4 GSM8K Completion Evaluation Test (8-GPU)
|
||||
|
||||
Tests DeepSeek-R1-MXFP4 quantized model with multiple configurations
|
||||
(basic, MTP, DP, TC) using few-shot completion benchmark on MI35x.
|
||||
|
||||
Registry: nightly-amd-8-gpu-mi35x-deepseek-r1-mxfp4 suite
|
||||
"""
|
||||
|
||||
import ast
|
||||
import os
|
||||
|
||||
# Set HF cache for MI35x
|
||||
os.environ.setdefault("HF_HOME", "/data2/models/huggingface")
|
||||
os.environ.setdefault("HF_HUB_CACHE", "/data2/models/huggingface/hub")
|
||||
|
||||
import re
|
||||
import time
|
||||
import unittest
|
||||
from dataclasses import dataclass
|
||||
from typing import List, Optional, Tuple
|
||||
|
||||
import numpy as np
|
||||
|
||||
from sglang.srt.utils import kill_process_tree
|
||||
from sglang.test.ci.ci_register import register_amd_ci
|
||||
from sglang.test.test_utils import (
|
||||
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||
DEFAULT_URL_FOR_TEST,
|
||||
is_in_ci,
|
||||
popen_launch_server,
|
||||
write_github_step_summary,
|
||||
)
|
||||
from sglang.utils import download_and_cache_file, read_jsonl
|
||||
|
||||
# Register for AMD CI - MI35x DeepSeek-R1-MXFP4 accuracy tests (~120 min)
|
||||
register_amd_ci(
|
||||
est_time=7200, suite="nightly-amd-8-gpu-mi35x-deepseek-r1-mxfp4", nightly=True
|
||||
)
|
||||
|
||||
INVALID = -9999999
|
||||
|
||||
# Model path configuration for MI35x DeepSeek-R1-MXFP4
|
||||
# Priority: 1) env var, 2) local path, 3) HuggingFace model ID
|
||||
DEEPSEEK_R1_MXFP4_LOCAL_PATH = "/data2/models/amd-DeepSeek-R1-MXFP4-Preview"
|
||||
DEEPSEEK_R1_MXFP4_HF_MODEL_ID = "amd/DeepSeek-R1-MXFP4-Preview"
|
||||
|
||||
|
||||
def get_model_path() -> str:
|
||||
"""Get effective model path: env var > local path > HF model ID."""
|
||||
env_path = os.environ.get("DEEPSEEK_R1_MXFP4_MODEL_PATH")
|
||||
if env_path:
|
||||
return env_path
|
||||
if os.path.exists(DEEPSEEK_R1_MXFP4_LOCAL_PATH):
|
||||
return DEEPSEEK_R1_MXFP4_LOCAL_PATH
|
||||
return DEEPSEEK_R1_MXFP4_HF_MODEL_ID
|
||||
|
||||
|
||||
@dataclass
|
||||
class ModelConfig:
|
||||
"""Configuration for a model to test."""
|
||||
|
||||
model_path: str
|
||||
tp_size: int = 8
|
||||
accuracy_threshold: float = 0.50
|
||||
other_args: Optional[List[str]] = None
|
||||
env_vars: Optional[dict] = None
|
||||
timeout: Optional[int] = None
|
||||
variant: Optional[str] = None
|
||||
|
||||
def __post_init__(self):
|
||||
if self.other_args is None:
|
||||
self.other_args = []
|
||||
if self.env_vars is None:
|
||||
self.env_vars = {}
|
||||
|
||||
def get_display_name(self) -> str:
|
||||
if self.variant:
|
||||
return f"{self.model_path} ({self.variant})"
|
||||
return self.model_path
|
||||
|
||||
|
||||
def get_mxfp4_models() -> List[ModelConfig]:
|
||||
"""Get DeepSeek-R1-MXFP4 model configurations for MI35x."""
|
||||
model_path = get_model_path()
|
||||
return [
|
||||
# DeepSeek-R1-MXFP4 basic
|
||||
ModelConfig(
|
||||
model_path=model_path,
|
||||
tp_size=8,
|
||||
accuracy_threshold=0.93,
|
||||
timeout=3600,
|
||||
variant="basic",
|
||||
other_args=[
|
||||
"--attention-backend",
|
||||
"aiter",
|
||||
"--chunked-prefill-size",
|
||||
"131072",
|
||||
"--disable-radix-cache",
|
||||
"--mem-fraction-static",
|
||||
"0.85",
|
||||
"--trust-remote-code",
|
||||
],
|
||||
env_vars={"SGLANG_USE_AITER": "1"},
|
||||
),
|
||||
# DeepSeek-R1-MXFP4 with MTP (EAGLE)
|
||||
ModelConfig(
|
||||
model_path=model_path,
|
||||
tp_size=8,
|
||||
accuracy_threshold=0.93,
|
||||
timeout=3600,
|
||||
variant="MTP",
|
||||
other_args=[
|
||||
"--chunked-prefill-size",
|
||||
"131072",
|
||||
"--speculative-algorithm",
|
||||
"EAGLE",
|
||||
"--speculative-num-steps",
|
||||
"3",
|
||||
"--speculative-eagle-topk",
|
||||
"1",
|
||||
"--speculative-num-draft-tokens",
|
||||
"4",
|
||||
"--mem-fraction-static",
|
||||
"0.7",
|
||||
"--trust-remote-code",
|
||||
],
|
||||
env_vars={"SGLANG_USE_AITER": "1"},
|
||||
),
|
||||
# Note: DP and TC variants are not supported for MXFP4 on MI35x
|
||||
]
|
||||
|
||||
|
||||
def get_one_example(lines, i, include_answer):
|
||||
"""Format a single GSM8K example."""
|
||||
ret = "Question: " + lines[i]["question"] + "\nAnswer:"
|
||||
if include_answer:
|
||||
ret += " " + lines[i]["answer"]
|
||||
return ret
|
||||
|
||||
|
||||
def get_few_shot_examples(lines, k):
|
||||
"""Get k few-shot examples for prompting."""
|
||||
ret = ""
|
||||
for i in range(k):
|
||||
ret += get_one_example(lines, i, True) + "\n\n"
|
||||
return ret
|
||||
|
||||
|
||||
def get_answer_value(answer_str):
|
||||
"""Extract numerical answer from response."""
|
||||
answer_str = answer_str.replace(",", "")
|
||||
numbers = re.findall(r"\d+", answer_str)
|
||||
if len(numbers) < 1:
|
||||
return INVALID
|
||||
try:
|
||||
return ast.literal_eval(numbers[-1])
|
||||
except SyntaxError:
|
||||
return INVALID
|
||||
|
||||
|
||||
def run_gsm8k_benchmark(
|
||||
base_url: str,
|
||||
num_questions: int = 200,
|
||||
num_shots: int = 5,
|
||||
parallel: int = 64,
|
||||
) -> Tuple[float, float, float]:
|
||||
"""Run GSM8K few-shot completion benchmark."""
|
||||
import sglang as sgl
|
||||
from sglang.lang.backend.runtime_endpoint import RuntimeEndpoint
|
||||
|
||||
url = "https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/test.jsonl"
|
||||
data_path = download_and_cache_file(url)
|
||||
lines = list(read_jsonl(data_path))
|
||||
|
||||
few_shot_examples = get_few_shot_examples(lines, num_shots)
|
||||
|
||||
questions = []
|
||||
labels = []
|
||||
for i in range(len(lines[:num_questions])):
|
||||
questions.append(get_one_example(lines, i, False))
|
||||
labels.append(get_answer_value(lines[i]["answer"]))
|
||||
assert all(l != INVALID for l in labels)
|
||||
arguments = [{"question": q} for q in questions]
|
||||
|
||||
@sgl.function
|
||||
def few_shot_gsm8k(s, question):
|
||||
s += few_shot_examples + question
|
||||
s += sgl.gen(
|
||||
"answer", max_tokens=512, stop=["Question", "Assistant:", "<|separator|>"]
|
||||
)
|
||||
|
||||
backend = RuntimeEndpoint(base_url)
|
||||
sgl.set_default_backend(backend)
|
||||
|
||||
tic = time.perf_counter()
|
||||
states = few_shot_gsm8k.run_batch(
|
||||
arguments, temperature=0, num_threads=parallel, progress_bar=True
|
||||
)
|
||||
latency = time.perf_counter() - tic
|
||||
|
||||
preds = [get_answer_value(states[i]["answer"]) for i in range(len(states))]
|
||||
acc = np.mean(np.array(preds) == np.array(labels))
|
||||
invalid = np.mean(np.array(preds) == INVALID)
|
||||
|
||||
return float(acc), float(invalid), float(latency)
|
||||
|
||||
|
||||
class TestDeepSeekR1MXFP4EvalMI35x(unittest.TestCase):
|
||||
"""DeepSeek-R1-MXFP4 GSM8K Completion Evaluation Test for AMD MI35x."""
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.models = get_mxfp4_models()
|
||||
cls.base_url = DEFAULT_URL_FOR_TEST
|
||||
cls.num_questions = int(os.environ.get("GSM8K_NUM_QUESTIONS", "200"))
|
||||
|
||||
def test_deepseek_r1_mxfp4_accuracy(self):
|
||||
"""Test DeepSeek-R1-MXFP4 models with GSM8K completion benchmark."""
|
||||
# Check if model exists
|
||||
model_path = get_model_path()
|
||||
is_local_path = model_path.startswith("/")
|
||||
if is_local_path and not os.path.exists(model_path):
|
||||
print(f"\n⏭️ SKIPPING: Local model not found at {model_path}")
|
||||
self.skipTest(f"Local model not found at {model_path}")
|
||||
return
|
||||
|
||||
if is_local_path:
|
||||
print(f"📁 Using local model: {model_path}")
|
||||
else:
|
||||
print(f"📥 Using HuggingFace model: {model_path}")
|
||||
|
||||
all_results = []
|
||||
summary = "### DeepSeek-R1-MXFP4 Models (MI35x)\n\n"
|
||||
summary += "| Model | Variant | TP | Accuracy | Threshold | Status |\n"
|
||||
summary += "| ----- | ------- | -- | -------- | --------- | ------ |\n"
|
||||
|
||||
for config in self.models:
|
||||
display_name = config.get_display_name()
|
||||
with self.subTest(model=display_name):
|
||||
print(f"\n{'='*60}")
|
||||
print(f"Testing: {display_name}")
|
||||
print(f"{'='*60}")
|
||||
|
||||
env = os.environ.copy()
|
||||
for key, value in config.env_vars.items():
|
||||
env[key] = value
|
||||
|
||||
other_args = list(config.other_args)
|
||||
other_args.extend(["--tp", str(config.tp_size)])
|
||||
timeout = config.timeout or DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH
|
||||
|
||||
try:
|
||||
process = popen_launch_server(
|
||||
model=config.model_path,
|
||||
base_url=self.base_url,
|
||||
timeout=timeout,
|
||||
other_args=other_args,
|
||||
env=env,
|
||||
)
|
||||
|
||||
try:
|
||||
acc, invalid, latency = run_gsm8k_benchmark(
|
||||
self.base_url, num_questions=self.num_questions
|
||||
)
|
||||
passed = acc >= config.accuracy_threshold
|
||||
status = "✅ PASS" if passed else "❌ FAIL"
|
||||
|
||||
all_results.append(
|
||||
{
|
||||
"model": display_name,
|
||||
"accuracy": acc,
|
||||
"passed": passed,
|
||||
}
|
||||
)
|
||||
summary += f"| {config.model_path} | {config.variant or 'N/A'} | {config.tp_size} | {acc:.3f} | {config.accuracy_threshold} | {status} |\n"
|
||||
|
||||
finally:
|
||||
kill_process_tree(process.pid)
|
||||
|
||||
except Exception as e:
|
||||
summary += f"| {config.model_path} | {config.variant or 'N/A'} | {config.tp_size} | N/A | {config.accuracy_threshold} | ❌ ERROR |\n"
|
||||
all_results.append(
|
||||
{
|
||||
"model": display_name,
|
||||
"accuracy": None,
|
||||
"passed": False,
|
||||
"error": str(e),
|
||||
}
|
||||
)
|
||||
|
||||
if is_in_ci():
|
||||
write_github_step_summary(summary)
|
||||
|
||||
failed = [r for r in all_results if not r["passed"]]
|
||||
if failed:
|
||||
raise AssertionError(f"Failed models: {[r['model'] for r in failed]}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
254
test/registered/amd/accuracy/mi35x/test_gpt_oss_eval_mi35x.py
Normal file
254
test/registered/amd/accuracy/mi35x/test_gpt_oss_eval_mi35x.py
Normal file
@@ -0,0 +1,254 @@
|
||||
"""MI35x GPT-OSS GSM8K Completion Evaluation Test (8-GPU)
|
||||
|
||||
Tests GPT-OSS models (openai/gpt-oss-20b, openai/gpt-oss-120b) using
|
||||
few-shot completion benchmark on MI35x.
|
||||
|
||||
Note: MI35x uses openai/* paths, not lmsys/* paths like MI300X.
|
||||
|
||||
Registry: nightly-amd-8-gpu-mi35x suite
|
||||
"""
|
||||
|
||||
import ast
|
||||
import os
|
||||
|
||||
# Set HF cache for MI35x
|
||||
os.environ.setdefault("HF_HOME", "/data2/models/huggingface")
|
||||
os.environ.setdefault("HF_HUB_CACHE", "/data2/models/huggingface/hub")
|
||||
|
||||
import re
|
||||
import time
|
||||
import unittest
|
||||
from dataclasses import dataclass
|
||||
from typing import List, Optional, Tuple
|
||||
|
||||
import numpy as np
|
||||
|
||||
from sglang.srt.utils import kill_process_tree
|
||||
from sglang.test.ci.ci_register import register_amd_ci
|
||||
from sglang.test.test_utils import (
|
||||
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||
DEFAULT_URL_FOR_TEST,
|
||||
is_in_ci,
|
||||
popen_launch_server,
|
||||
write_github_step_summary,
|
||||
)
|
||||
from sglang.utils import download_and_cache_file, read_jsonl
|
||||
|
||||
# Register for AMD CI - MI35x GPT-OSS accuracy tests (~30 min)
|
||||
register_amd_ci(est_time=1800, suite="nightly-amd-8-gpu-mi35x", nightly=True)
|
||||
|
||||
INVALID = -9999999
|
||||
|
||||
|
||||
@dataclass
|
||||
class ModelConfig:
|
||||
"""Configuration for a model to test."""
|
||||
|
||||
model_path: str
|
||||
tp_size: int = 8
|
||||
accuracy_threshold: float = 0.50
|
||||
other_args: Optional[List[str]] = None
|
||||
env_vars: Optional[dict] = None
|
||||
timeout: Optional[int] = None
|
||||
|
||||
def __post_init__(self):
|
||||
if self.other_args is None:
|
||||
self.other_args = []
|
||||
if self.env_vars is None:
|
||||
self.env_vars = {}
|
||||
|
||||
|
||||
# GPT-OSS models for MI35x (different paths from MI300X)
|
||||
MI35X_GPT_OSS_MODELS = [
|
||||
ModelConfig(
|
||||
model_path="openai/gpt-oss-20b",
|
||||
tp_size=8,
|
||||
accuracy_threshold=0.47,
|
||||
other_args=[
|
||||
"--chunked-prefill-size",
|
||||
"130172",
|
||||
"--max-running-requests",
|
||||
"128",
|
||||
"--mem-fraction-static",
|
||||
"0.85",
|
||||
"--attention-backend",
|
||||
"triton",
|
||||
"--trust-remote-code",
|
||||
],
|
||||
env_vars={"SGLANG_USE_AITER": "1"},
|
||||
),
|
||||
ModelConfig(
|
||||
model_path="openai/gpt-oss-120b",
|
||||
tp_size=8,
|
||||
accuracy_threshold=0.79,
|
||||
timeout=900,
|
||||
other_args=[
|
||||
"--chunked-prefill-size",
|
||||
"130172",
|
||||
"--max-running-requests",
|
||||
"128",
|
||||
"--mem-fraction-static",
|
||||
"0.85",
|
||||
"--attention-backend",
|
||||
"triton",
|
||||
"--trust-remote-code",
|
||||
],
|
||||
env_vars={"SGLANG_USE_AITER": "1"},
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def get_one_example(lines, i, include_answer):
|
||||
"""Format a single GSM8K example."""
|
||||
ret = "Question: " + lines[i]["question"] + "\nAnswer:"
|
||||
if include_answer:
|
||||
ret += " " + lines[i]["answer"]
|
||||
return ret
|
||||
|
||||
|
||||
def get_few_shot_examples(lines, k):
|
||||
"""Get k few-shot examples for prompting."""
|
||||
ret = ""
|
||||
for i in range(k):
|
||||
ret += get_one_example(lines, i, True) + "\n\n"
|
||||
return ret
|
||||
|
||||
|
||||
def get_answer_value(answer_str):
|
||||
"""Extract numerical answer from response."""
|
||||
answer_str = answer_str.replace(",", "")
|
||||
numbers = re.findall(r"\d+", answer_str)
|
||||
if len(numbers) < 1:
|
||||
return INVALID
|
||||
try:
|
||||
return ast.literal_eval(numbers[-1])
|
||||
except SyntaxError:
|
||||
return INVALID
|
||||
|
||||
|
||||
def run_gsm8k_benchmark(
|
||||
base_url: str,
|
||||
num_questions: int = 200,
|
||||
num_shots: int = 5,
|
||||
parallel: int = 64,
|
||||
) -> Tuple[float, float, float]:
|
||||
"""Run GSM8K few-shot completion benchmark."""
|
||||
import sglang as sgl
|
||||
from sglang.lang.backend.runtime_endpoint import RuntimeEndpoint
|
||||
|
||||
url = "https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/test.jsonl"
|
||||
data_path = download_and_cache_file(url)
|
||||
lines = list(read_jsonl(data_path))
|
||||
|
||||
few_shot_examples = get_few_shot_examples(lines, num_shots)
|
||||
|
||||
questions = []
|
||||
labels = []
|
||||
for i in range(len(lines[:num_questions])):
|
||||
questions.append(get_one_example(lines, i, False))
|
||||
labels.append(get_answer_value(lines[i]["answer"]))
|
||||
assert all(l != INVALID for l in labels)
|
||||
arguments = [{"question": q} for q in questions]
|
||||
|
||||
@sgl.function
|
||||
def few_shot_gsm8k(s, question):
|
||||
s += few_shot_examples + question
|
||||
s += sgl.gen(
|
||||
"answer", max_tokens=512, stop=["Question", "Assistant:", "<|separator|>"]
|
||||
)
|
||||
|
||||
backend = RuntimeEndpoint(base_url)
|
||||
sgl.set_default_backend(backend)
|
||||
|
||||
tic = time.perf_counter()
|
||||
states = few_shot_gsm8k.run_batch(
|
||||
arguments, temperature=0, num_threads=parallel, progress_bar=True
|
||||
)
|
||||
latency = time.perf_counter() - tic
|
||||
|
||||
preds = [get_answer_value(states[i]["answer"]) for i in range(len(states))]
|
||||
acc = np.mean(np.array(preds) == np.array(labels))
|
||||
invalid = np.mean(np.array(preds) == INVALID)
|
||||
|
||||
return float(acc), float(invalid), float(latency)
|
||||
|
||||
|
||||
class TestGptOssEvalMI35x(unittest.TestCase):
|
||||
"""GPT-OSS GSM8K Completion Evaluation Test for AMD MI35x."""
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.models = MI35X_GPT_OSS_MODELS
|
||||
cls.base_url = DEFAULT_URL_FOR_TEST
|
||||
cls.num_questions = int(os.environ.get("GSM8K_NUM_QUESTIONS", "200"))
|
||||
|
||||
def test_gpt_oss_accuracy(self):
|
||||
"""Test GPT-OSS models with GSM8K completion benchmark."""
|
||||
all_results = []
|
||||
summary = "### GPT-OSS Models (MI35x)\n\n"
|
||||
summary += "| Model | TP | Accuracy | Threshold | Status |\n"
|
||||
summary += "| ----- | -- | -------- | --------- | ------ |\n"
|
||||
|
||||
for config in self.models:
|
||||
with self.subTest(model=config.model_path):
|
||||
print(f"\n{'='*60}")
|
||||
print(f"Testing: {config.model_path}")
|
||||
print(f"{'='*60}")
|
||||
|
||||
env = os.environ.copy()
|
||||
for key, value in config.env_vars.items():
|
||||
env[key] = value
|
||||
|
||||
other_args = list(config.other_args)
|
||||
other_args.extend(["--tp", str(config.tp_size)])
|
||||
timeout = config.timeout or DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH
|
||||
|
||||
try:
|
||||
process = popen_launch_server(
|
||||
model=config.model_path,
|
||||
base_url=self.base_url,
|
||||
timeout=timeout,
|
||||
other_args=other_args,
|
||||
env=env,
|
||||
)
|
||||
|
||||
try:
|
||||
acc, invalid, latency = run_gsm8k_benchmark(
|
||||
self.base_url, num_questions=self.num_questions
|
||||
)
|
||||
passed = acc >= config.accuracy_threshold
|
||||
status = "✅ PASS" if passed else "❌ FAIL"
|
||||
|
||||
all_results.append(
|
||||
{
|
||||
"model": config.model_path,
|
||||
"accuracy": acc,
|
||||
"passed": passed,
|
||||
}
|
||||
)
|
||||
summary += f"| {config.model_path} | {config.tp_size} | {acc:.3f} | {config.accuracy_threshold} | {status} |\n"
|
||||
|
||||
finally:
|
||||
kill_process_tree(process.pid)
|
||||
|
||||
except Exception as e:
|
||||
summary += f"| {config.model_path} | {config.tp_size} | N/A | {config.accuracy_threshold} | ❌ ERROR |\n"
|
||||
all_results.append(
|
||||
{
|
||||
"model": config.model_path,
|
||||
"accuracy": None,
|
||||
"passed": False,
|
||||
"error": str(e),
|
||||
}
|
||||
)
|
||||
|
||||
if is_in_ci():
|
||||
write_github_step_summary(summary)
|
||||
|
||||
failed = [r for r in all_results if not r["passed"]]
|
||||
if failed:
|
||||
raise AssertionError(f"Failed models: {[r['model'] for r in failed]}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
162
test/registered/amd/accuracy/mi35x/test_grok1_int4_eval_mi35x.py
Normal file
162
test/registered/amd/accuracy/mi35x/test_grok1_int4_eval_mi35x.py
Normal file
@@ -0,0 +1,162 @@
|
||||
"""MI35x GROK1-INT4 GSM8K Completion Evaluation Test (8-GPU)
|
||||
|
||||
Tests Grok-1 INT4 (W4A8KV8) model using few-shot completion benchmark on MI35x.
|
||||
|
||||
Registry: nightly-amd-accuracy-8-gpu-mi35x-grok1-int4 suite
|
||||
"""
|
||||
|
||||
import ast
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
import unittest
|
||||
|
||||
import numpy as np
|
||||
|
||||
from sglang.srt.utils import kill_process_tree
|
||||
from sglang.test.ci.ci_register import register_amd_ci
|
||||
from sglang.test.test_utils import (
|
||||
DEFAULT_URL_FOR_TEST,
|
||||
is_in_ci,
|
||||
popen_launch_server,
|
||||
write_github_step_summary,
|
||||
)
|
||||
from sglang.utils import download_and_cache_file, read_jsonl
|
||||
|
||||
# Register for AMD CI - GROK1-INT4 accuracy tests on MI35x (~25 min)
|
||||
register_amd_ci(
|
||||
est_time=1500, suite="nightly-amd-accuracy-8-gpu-mi35x-grok1-int4", nightly=True
|
||||
)
|
||||
|
||||
INVALID = -9999999
|
||||
|
||||
GROK1_INT4_MODEL_PATH = os.environ.get("GROK1_INT4_MODEL_PATH", "amd/grok-1-W4A8KV8")
|
||||
GROK1_TOKENIZER_PATH = os.environ.get("GROK1_TOKENIZER_PATH", "Xenova/grok-1-tokenizer")
|
||||
|
||||
|
||||
def get_one_example(lines, i, include_answer):
|
||||
ret = "Question: " + lines[i]["question"] + "\nAnswer:"
|
||||
if include_answer:
|
||||
ret += " " + lines[i]["answer"]
|
||||
return ret
|
||||
|
||||
|
||||
def get_few_shot_examples(lines, k):
|
||||
ret = ""
|
||||
for i in range(k):
|
||||
ret += get_one_example(lines, i, True) + "\n\n"
|
||||
return ret
|
||||
|
||||
|
||||
def get_answer_value(answer_str):
|
||||
answer_str = answer_str.replace(",", "")
|
||||
numbers = re.findall(r"\d+", answer_str)
|
||||
if len(numbers) < 1:
|
||||
return INVALID
|
||||
try:
|
||||
return ast.literal_eval(numbers[-1])
|
||||
except SyntaxError:
|
||||
return INVALID
|
||||
|
||||
|
||||
def run_gsm8k_benchmark(base_url, num_questions=200, num_shots=5, parallel=64):
|
||||
import sglang as sgl
|
||||
from sglang.lang.backend.runtime_endpoint import RuntimeEndpoint
|
||||
|
||||
url = "https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/test.jsonl"
|
||||
data_path = download_and_cache_file(url)
|
||||
lines = list(read_jsonl(data_path))
|
||||
|
||||
few_shot_examples = get_few_shot_examples(lines, num_shots)
|
||||
questions = []
|
||||
labels = []
|
||||
for i in range(len(lines[:num_questions])):
|
||||
questions.append(get_one_example(lines, i, False))
|
||||
labels.append(get_answer_value(lines[i]["answer"]))
|
||||
arguments = [{"question": q} for q in questions]
|
||||
|
||||
@sgl.function
|
||||
def few_shot_gsm8k(s, question):
|
||||
s += few_shot_examples + question
|
||||
s += sgl.gen(
|
||||
"answer", max_tokens=512, stop=["Question", "Assistant:", "<|separator|>"]
|
||||
)
|
||||
|
||||
backend = RuntimeEndpoint(base_url)
|
||||
sgl.set_default_backend(backend)
|
||||
|
||||
tic = time.perf_counter()
|
||||
states = few_shot_gsm8k.run_batch(
|
||||
arguments, temperature=0, num_threads=parallel, progress_bar=True
|
||||
)
|
||||
latency = time.perf_counter() - tic
|
||||
|
||||
preds = [get_answer_value(states[i]["answer"]) for i in range(len(states))]
|
||||
acc = np.mean(np.array(preds) == np.array(labels))
|
||||
return float(acc), float(latency)
|
||||
|
||||
|
||||
class TestGrok1INT4EvalMI35x(unittest.TestCase):
|
||||
"""GROK1-INT4 GSM8K Completion Evaluation Test for MI35x."""
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.base_url = DEFAULT_URL_FOR_TEST
|
||||
cls.num_questions = int(os.environ.get("GSM8K_NUM_QUESTIONS", "200"))
|
||||
cls.accuracy_threshold = 0.80
|
||||
|
||||
def test_grok1_int4_accuracy(self):
|
||||
"""Test Grok-1 INT4 with GSM8K completion benchmark."""
|
||||
env = os.environ.copy()
|
||||
env["RCCL_MSCCL_ENABLE"] = "0"
|
||||
env["SGLANG_USE_AITER"] = "1"
|
||||
env["SGLANG_INT4_WEIGHT"] = "1"
|
||||
|
||||
other_args = [
|
||||
"--tp",
|
||||
"8",
|
||||
"--quantization",
|
||||
"fp8",
|
||||
"--attention-backend",
|
||||
"aiter",
|
||||
"--mem-fraction-static",
|
||||
"0.85",
|
||||
"--tokenizer-path",
|
||||
GROK1_TOKENIZER_PATH,
|
||||
"--trust-remote-code",
|
||||
]
|
||||
|
||||
process = popen_launch_server(
|
||||
model=GROK1_INT4_MODEL_PATH,
|
||||
base_url=self.base_url,
|
||||
timeout=3600,
|
||||
other_args=other_args,
|
||||
env=env,
|
||||
)
|
||||
|
||||
try:
|
||||
acc, latency = run_gsm8k_benchmark(
|
||||
self.base_url, num_questions=self.num_questions
|
||||
)
|
||||
passed = acc >= self.accuracy_threshold
|
||||
status = "✅ PASS" if passed else "❌ FAIL"
|
||||
|
||||
summary = f"### GROK1-INT4 (MI35x)\n\n"
|
||||
summary += f"| Model | Accuracy | Threshold | Status |\n"
|
||||
summary += f"| ----- | -------- | --------- | ------ |\n"
|
||||
summary += f"| {GROK1_INT4_MODEL_PATH} | {acc:.3f} | {self.accuracy_threshold} | {status} |\n"
|
||||
|
||||
if is_in_ci():
|
||||
write_github_step_summary(summary)
|
||||
|
||||
self.assertGreaterEqual(
|
||||
acc,
|
||||
self.accuracy_threshold,
|
||||
f"Accuracy {acc:.3f} below threshold {self.accuracy_threshold}",
|
||||
)
|
||||
finally:
|
||||
kill_process_tree(process.pid)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
164
test/registered/amd/accuracy/mi35x/test_grok2_eval_mi35x.py
Normal file
164
test/registered/amd/accuracy/mi35x/test_grok2_eval_mi35x.py
Normal file
@@ -0,0 +1,164 @@
|
||||
"""MI35x GROK2 GSM8K Completion Evaluation Test (8-GPU)
|
||||
|
||||
Tests Grok-2 model using few-shot completion benchmark on MI35x.
|
||||
|
||||
Registry: nightly-amd-accuracy-8-gpu-mi35x-grok2 suite
|
||||
"""
|
||||
|
||||
import ast
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
import unittest
|
||||
|
||||
import numpy as np
|
||||
|
||||
from sglang.srt.utils import kill_process_tree
|
||||
from sglang.test.ci.ci_register import register_amd_ci
|
||||
from sglang.test.test_utils import (
|
||||
DEFAULT_URL_FOR_TEST,
|
||||
is_in_ci,
|
||||
popen_launch_server,
|
||||
write_github_step_summary,
|
||||
)
|
||||
from sglang.utils import download_and_cache_file, read_jsonl
|
||||
|
||||
# Register for AMD CI - GROK2 accuracy tests on MI35x (~25 min)
|
||||
register_amd_ci(
|
||||
est_time=1500, suite="nightly-amd-accuracy-8-gpu-mi35x-grok2", nightly=True
|
||||
)
|
||||
|
||||
INVALID = -9999999
|
||||
|
||||
GROK2_MODEL_PATH = os.environ.get("GROK2_MODEL_PATH", "xai-org/grok-2")
|
||||
GROK2_TOKENIZER_PATH = os.environ.get(
|
||||
"GROK2_TOKENIZER_PATH", "alvarobartt/grok-2-tokenizer"
|
||||
)
|
||||
|
||||
|
||||
def get_one_example(lines, i, include_answer):
|
||||
ret = "Question: " + lines[i]["question"] + "\nAnswer:"
|
||||
if include_answer:
|
||||
ret += " " + lines[i]["answer"]
|
||||
return ret
|
||||
|
||||
|
||||
def get_few_shot_examples(lines, k):
|
||||
ret = ""
|
||||
for i in range(k):
|
||||
ret += get_one_example(lines, i, True) + "\n\n"
|
||||
return ret
|
||||
|
||||
|
||||
def get_answer_value(answer_str):
|
||||
answer_str = answer_str.replace(",", "")
|
||||
numbers = re.findall(r"\d+", answer_str)
|
||||
if len(numbers) < 1:
|
||||
return INVALID
|
||||
try:
|
||||
return ast.literal_eval(numbers[-1])
|
||||
except SyntaxError:
|
||||
return INVALID
|
||||
|
||||
|
||||
def run_gsm8k_benchmark(base_url, num_questions=200, num_shots=5, parallel=64):
|
||||
import sglang as sgl
|
||||
from sglang.lang.backend.runtime_endpoint import RuntimeEndpoint
|
||||
|
||||
url = "https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/test.jsonl"
|
||||
data_path = download_and_cache_file(url)
|
||||
lines = list(read_jsonl(data_path))
|
||||
|
||||
few_shot_examples = get_few_shot_examples(lines, num_shots)
|
||||
questions = []
|
||||
labels = []
|
||||
for i in range(len(lines[:num_questions])):
|
||||
questions.append(get_one_example(lines, i, False))
|
||||
labels.append(get_answer_value(lines[i]["answer"]))
|
||||
arguments = [{"question": q} for q in questions]
|
||||
|
||||
@sgl.function
|
||||
def few_shot_gsm8k(s, question):
|
||||
s += few_shot_examples + question
|
||||
s += sgl.gen(
|
||||
"answer", max_tokens=512, stop=["Question", "Assistant:", "<|separator|>"]
|
||||
)
|
||||
|
||||
backend = RuntimeEndpoint(base_url)
|
||||
sgl.set_default_backend(backend)
|
||||
|
||||
tic = time.perf_counter()
|
||||
states = few_shot_gsm8k.run_batch(
|
||||
arguments, temperature=0, num_threads=parallel, progress_bar=True
|
||||
)
|
||||
latency = time.perf_counter() - tic
|
||||
|
||||
preds = [get_answer_value(states[i]["answer"]) for i in range(len(states))]
|
||||
acc = np.mean(np.array(preds) == np.array(labels))
|
||||
return float(acc), float(latency)
|
||||
|
||||
|
||||
class TestGrok2EvalMI35x(unittest.TestCase):
|
||||
"""GROK2 GSM8K Completion Evaluation Test for MI35x."""
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.base_url = DEFAULT_URL_FOR_TEST
|
||||
cls.num_questions = int(os.environ.get("GSM8K_NUM_QUESTIONS", "200"))
|
||||
cls.accuracy_threshold = 0.915
|
||||
|
||||
def test_grok2_accuracy(self):
|
||||
"""Test Grok-2 with GSM8K completion benchmark."""
|
||||
env = os.environ.copy()
|
||||
env["RCCL_MSCCL_ENABLE"] = "0"
|
||||
env["SGLANG_USE_AITER"] = "1"
|
||||
env["SGLANG_INT4_WEIGHT"] = "0"
|
||||
|
||||
other_args = [
|
||||
"--tp",
|
||||
"8",
|
||||
"--quantization",
|
||||
"fp8",
|
||||
"--attention-backend",
|
||||
"aiter",
|
||||
"--mem-fraction-static",
|
||||
"0.85",
|
||||
"--tokenizer-path",
|
||||
GROK2_TOKENIZER_PATH,
|
||||
"--trust-remote-code",
|
||||
]
|
||||
|
||||
process = popen_launch_server(
|
||||
model=GROK2_MODEL_PATH,
|
||||
base_url=self.base_url,
|
||||
timeout=3600,
|
||||
other_args=other_args,
|
||||
env=env,
|
||||
)
|
||||
|
||||
try:
|
||||
acc, latency = run_gsm8k_benchmark(
|
||||
self.base_url, num_questions=self.num_questions
|
||||
)
|
||||
passed = acc >= self.accuracy_threshold
|
||||
status = "✅ PASS" if passed else "❌ FAIL"
|
||||
|
||||
summary = f"### GROK2 (MI35x)\n\n"
|
||||
summary += f"| Model | Accuracy | Threshold | Status |\n"
|
||||
summary += f"| ----- | -------- | --------- | ------ |\n"
|
||||
summary += f"| {GROK2_MODEL_PATH} | {acc:.3f} | {self.accuracy_threshold} | {status} |\n"
|
||||
|
||||
if is_in_ci():
|
||||
write_github_step_summary(summary)
|
||||
|
||||
self.assertGreaterEqual(
|
||||
acc,
|
||||
self.accuracy_threshold,
|
||||
f"Accuracy {acc:.3f} below threshold {self.accuracy_threshold}",
|
||||
)
|
||||
finally:
|
||||
kill_process_tree(process.pid)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
309
test/registered/amd/accuracy/test_deepseek_r1_eval_amd.py
Normal file
309
test/registered/amd/accuracy/test_deepseek_r1_eval_amd.py
Normal file
@@ -0,0 +1,309 @@
|
||||
"""AMD DeepSeek-R1 GSM8K Completion Evaluation Test (8-GPU)
|
||||
|
||||
Tests DeepSeek-R1-0528 with multiple configurations (basic, MTP, DP, TC)
|
||||
using few-shot completion benchmark on MI300X.
|
||||
|
||||
Registry: nightly-amd-8-gpu-deepseek-r1 suite
|
||||
"""
|
||||
|
||||
import ast
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
import unittest
|
||||
from dataclasses import dataclass
|
||||
from typing import List, Optional, Tuple
|
||||
|
||||
import numpy as np
|
||||
|
||||
from sglang.srt.utils import kill_process_tree
|
||||
from sglang.test.ci.ci_register import register_amd_ci
|
||||
from sglang.test.test_utils import (
|
||||
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||
DEFAULT_URL_FOR_TEST,
|
||||
is_in_ci,
|
||||
popen_launch_server,
|
||||
write_github_step_summary,
|
||||
)
|
||||
from sglang.utils import download_and_cache_file, read_jsonl
|
||||
|
||||
# Register for AMD CI - DeepSeek-R1 accuracy tests (~120 min)
|
||||
register_amd_ci(
|
||||
est_time=7200, suite="nightly-amd-accuracy-8-gpu-deepseek-r1", nightly=True
|
||||
)
|
||||
|
||||
INVALID = -9999999
|
||||
|
||||
|
||||
@dataclass
|
||||
class ModelConfig:
|
||||
"""Configuration for a model to test."""
|
||||
|
||||
model_path: str
|
||||
tp_size: int = 8
|
||||
accuracy_threshold: float = 0.50
|
||||
other_args: Optional[List[str]] = None
|
||||
env_vars: Optional[dict] = None
|
||||
timeout: Optional[int] = None
|
||||
variant: Optional[str] = None
|
||||
|
||||
def __post_init__(self):
|
||||
if self.other_args is None:
|
||||
self.other_args = []
|
||||
if self.env_vars is None:
|
||||
self.env_vars = {}
|
||||
|
||||
def get_display_name(self) -> str:
|
||||
if self.variant:
|
||||
return f"{self.model_path} ({self.variant})"
|
||||
return self.model_path
|
||||
|
||||
|
||||
# DeepSeek-R1 models for MI300X
|
||||
DEEPSEEK_R1_MODELS = [
|
||||
# DeepSeek-R1-0528 basic
|
||||
ModelConfig(
|
||||
model_path="deepseek-ai/DeepSeek-R1-0528",
|
||||
tp_size=8,
|
||||
accuracy_threshold=0.93,
|
||||
timeout=3600,
|
||||
variant="basic",
|
||||
other_args=[
|
||||
"--attention-backend",
|
||||
"aiter",
|
||||
"--chunked-prefill-size",
|
||||
"131072",
|
||||
"--disable-radix-cache",
|
||||
"--mem-fraction-static",
|
||||
"0.85",
|
||||
"--trust-remote-code",
|
||||
],
|
||||
env_vars={"SGLANG_USE_AITER": "1"},
|
||||
),
|
||||
# DeepSeek-R1-0528 with MTP (EAGLE)
|
||||
ModelConfig(
|
||||
model_path="deepseek-ai/DeepSeek-R1-0528",
|
||||
tp_size=8,
|
||||
accuracy_threshold=0.93,
|
||||
timeout=3600,
|
||||
variant="MTP",
|
||||
other_args=[
|
||||
"--chunked-prefill-size",
|
||||
"131072",
|
||||
"--speculative-algorithm",
|
||||
"EAGLE",
|
||||
"--speculative-num-steps",
|
||||
"3",
|
||||
"--speculative-eagle-topk",
|
||||
"1",
|
||||
"--speculative-num-draft-tokens",
|
||||
"4",
|
||||
"--mem-fraction-static",
|
||||
"0.7",
|
||||
"--trust-remote-code",
|
||||
],
|
||||
env_vars={"SGLANG_USE_AITER": "1"},
|
||||
),
|
||||
# DeepSeek-R1-0528 with DP attention
|
||||
ModelConfig(
|
||||
model_path="deepseek-ai/DeepSeek-R1-0528",
|
||||
tp_size=8,
|
||||
accuracy_threshold=0.93,
|
||||
timeout=3600,
|
||||
variant="DP",
|
||||
other_args=[
|
||||
"--chunked-prefill-size",
|
||||
"131072",
|
||||
"--dp-size",
|
||||
"8",
|
||||
"--enable-dp-attention",
|
||||
"--mem-fraction-static",
|
||||
"0.85",
|
||||
"--trust-remote-code",
|
||||
],
|
||||
env_vars={
|
||||
"SGLANG_USE_ROCM700A": "1",
|
||||
"SGLANG_USE_AITER": "1",
|
||||
},
|
||||
),
|
||||
# DeepSeek-R1-0528 with torch compile
|
||||
ModelConfig(
|
||||
model_path="deepseek-ai/DeepSeek-R1-0528",
|
||||
tp_size=8,
|
||||
accuracy_threshold=0.93,
|
||||
timeout=7200,
|
||||
variant="TC",
|
||||
other_args=[
|
||||
"--chunked-prefill-size",
|
||||
"131072",
|
||||
"--mem-fraction-static",
|
||||
"0.70",
|
||||
"--cuda-graph-max-bs",
|
||||
"8",
|
||||
"--enable-torch-compile",
|
||||
"--disable-cuda-graph",
|
||||
"--trust-remote-code",
|
||||
],
|
||||
env_vars={
|
||||
"SGLANG_USE_ROCM700A": "1",
|
||||
"SGLANG_USE_AITER": "1",
|
||||
},
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def get_one_example(lines, i, include_answer):
|
||||
"""Format a single GSM8K example."""
|
||||
ret = "Question: " + lines[i]["question"] + "\nAnswer:"
|
||||
if include_answer:
|
||||
ret += " " + lines[i]["answer"]
|
||||
return ret
|
||||
|
||||
|
||||
def get_few_shot_examples(lines, k):
|
||||
"""Get k few-shot examples for prompting."""
|
||||
ret = ""
|
||||
for i in range(k):
|
||||
ret += get_one_example(lines, i, True) + "\n\n"
|
||||
return ret
|
||||
|
||||
|
||||
def get_answer_value(answer_str):
|
||||
"""Extract numerical answer from response."""
|
||||
answer_str = answer_str.replace(",", "")
|
||||
numbers = re.findall(r"\d+", answer_str)
|
||||
if len(numbers) < 1:
|
||||
return INVALID
|
||||
try:
|
||||
return ast.literal_eval(numbers[-1])
|
||||
except SyntaxError:
|
||||
return INVALID
|
||||
|
||||
|
||||
def run_gsm8k_benchmark(
|
||||
base_url: str,
|
||||
num_questions: int = 200,
|
||||
num_shots: int = 5,
|
||||
parallel: int = 64,
|
||||
) -> Tuple[float, float, float]:
|
||||
"""Run GSM8K few-shot completion benchmark."""
|
||||
import sglang as sgl
|
||||
from sglang.lang.backend.runtime_endpoint import RuntimeEndpoint
|
||||
|
||||
url = "https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/test.jsonl"
|
||||
data_path = download_and_cache_file(url)
|
||||
lines = list(read_jsonl(data_path))
|
||||
|
||||
few_shot_examples = get_few_shot_examples(lines, num_shots)
|
||||
|
||||
questions = []
|
||||
labels = []
|
||||
for i in range(len(lines[:num_questions])):
|
||||
questions.append(get_one_example(lines, i, False))
|
||||
labels.append(get_answer_value(lines[i]["answer"]))
|
||||
assert all(l != INVALID for l in labels)
|
||||
arguments = [{"question": q} for q in questions]
|
||||
|
||||
@sgl.function
|
||||
def few_shot_gsm8k(s, question):
|
||||
s += few_shot_examples + question
|
||||
s += sgl.gen(
|
||||
"answer", max_tokens=512, stop=["Question", "Assistant:", "<|separator|>"]
|
||||
)
|
||||
|
||||
backend = RuntimeEndpoint(base_url)
|
||||
sgl.set_default_backend(backend)
|
||||
|
||||
tic = time.perf_counter()
|
||||
states = few_shot_gsm8k.run_batch(
|
||||
arguments, temperature=0, num_threads=parallel, progress_bar=True
|
||||
)
|
||||
latency = time.perf_counter() - tic
|
||||
|
||||
preds = [get_answer_value(states[i]["answer"]) for i in range(len(states))]
|
||||
acc = np.mean(np.array(preds) == np.array(labels))
|
||||
invalid = np.mean(np.array(preds) == INVALID)
|
||||
|
||||
return float(acc), float(invalid), float(latency)
|
||||
|
||||
|
||||
class TestDeepSeekR1EvalAMD(unittest.TestCase):
|
||||
"""DeepSeek-R1 GSM8K Completion Evaluation Test for AMD MI300X."""
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.models = DEEPSEEK_R1_MODELS
|
||||
cls.base_url = DEFAULT_URL_FOR_TEST
|
||||
cls.num_questions = int(os.environ.get("GSM8K_NUM_QUESTIONS", "200"))
|
||||
|
||||
def test_deepseek_r1_accuracy(self):
|
||||
"""Test DeepSeek-R1 models with GSM8K completion benchmark."""
|
||||
all_results = []
|
||||
summary = "### DeepSeek-R1 Models (MI300X)\n\n"
|
||||
summary += "| Model | Variant | TP | Accuracy | Threshold | Status |\n"
|
||||
summary += "| ----- | ------- | -- | -------- | --------- | ------ |\n"
|
||||
|
||||
for config in self.models:
|
||||
display_name = config.get_display_name()
|
||||
with self.subTest(model=display_name):
|
||||
print(f"\n{'='*60}")
|
||||
print(f"Testing: {display_name}")
|
||||
print(f"{'='*60}")
|
||||
|
||||
env = os.environ.copy()
|
||||
for key, value in config.env_vars.items():
|
||||
env[key] = value
|
||||
|
||||
other_args = list(config.other_args)
|
||||
other_args.extend(["--tp", str(config.tp_size)])
|
||||
timeout = config.timeout or DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH
|
||||
|
||||
try:
|
||||
process = popen_launch_server(
|
||||
model=config.model_path,
|
||||
base_url=self.base_url,
|
||||
timeout=timeout,
|
||||
other_args=other_args,
|
||||
env=env,
|
||||
)
|
||||
|
||||
try:
|
||||
acc, invalid, latency = run_gsm8k_benchmark(
|
||||
self.base_url, num_questions=self.num_questions
|
||||
)
|
||||
passed = acc >= config.accuracy_threshold
|
||||
status = "✅ PASS" if passed else "❌ FAIL"
|
||||
|
||||
all_results.append(
|
||||
{
|
||||
"model": display_name,
|
||||
"accuracy": acc,
|
||||
"passed": passed,
|
||||
}
|
||||
)
|
||||
summary += f"| {config.model_path} | {config.variant or 'N/A'} | {config.tp_size} | {acc:.3f} | {config.accuracy_threshold} | {status} |\n"
|
||||
|
||||
finally:
|
||||
kill_process_tree(process.pid)
|
||||
|
||||
except Exception as e:
|
||||
summary += f"| {config.model_path} | {config.variant or 'N/A'} | {config.tp_size} | N/A | {config.accuracy_threshold} | ❌ ERROR |\n"
|
||||
all_results.append(
|
||||
{
|
||||
"model": display_name,
|
||||
"accuracy": None,
|
||||
"passed": False,
|
||||
"error": str(e),
|
||||
}
|
||||
)
|
||||
|
||||
if is_in_ci():
|
||||
write_github_step_summary(summary)
|
||||
|
||||
failed = [r for r in all_results if not r["passed"]]
|
||||
if failed:
|
||||
raise AssertionError(f"Failed models: {[r['model'] for r in failed]}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
159
test/registered/amd/accuracy/test_deepseek_v31_eval_amd.py
Normal file
159
test/registered/amd/accuracy/test_deepseek_v31_eval_amd.py
Normal file
@@ -0,0 +1,159 @@
|
||||
"""AMD DeepSeek-V3.1 GSM8K Completion Evaluation Test (8-GPU)
|
||||
|
||||
Tests DeepSeek-V3.1 model using few-shot completion benchmark on MI300X.
|
||||
|
||||
Registry: nightly-amd-8-gpu-deepseek-v31 suite
|
||||
"""
|
||||
|
||||
import ast
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
import unittest
|
||||
|
||||
import numpy as np
|
||||
|
||||
from sglang.srt.utils import kill_process_tree
|
||||
from sglang.test.ci.ci_register import register_amd_ci
|
||||
from sglang.test.test_utils import (
|
||||
DEFAULT_URL_FOR_TEST,
|
||||
is_in_ci,
|
||||
popen_launch_server,
|
||||
write_github_step_summary,
|
||||
)
|
||||
from sglang.utils import download_and_cache_file, read_jsonl
|
||||
|
||||
# Register for AMD CI - DeepSeek-V3.1 accuracy tests (~60 min)
|
||||
register_amd_ci(
|
||||
est_time=3600, suite="nightly-amd-accuracy-8-gpu-deepseek-v31", nightly=True
|
||||
)
|
||||
|
||||
INVALID = -9999999
|
||||
|
||||
DEEPSEEK_V31_MODEL_PATH = os.environ.get(
|
||||
"DEEPSEEK_V31_MODEL_PATH", "deepseek-ai/DeepSeek-V3-0324"
|
||||
)
|
||||
|
||||
|
||||
def get_one_example(lines, i, include_answer):
|
||||
ret = "Question: " + lines[i]["question"] + "\nAnswer:"
|
||||
if include_answer:
|
||||
ret += " " + lines[i]["answer"]
|
||||
return ret
|
||||
|
||||
|
||||
def get_few_shot_examples(lines, k):
|
||||
ret = ""
|
||||
for i in range(k):
|
||||
ret += get_one_example(lines, i, True) + "\n\n"
|
||||
return ret
|
||||
|
||||
|
||||
def get_answer_value(answer_str):
|
||||
answer_str = answer_str.replace(",", "")
|
||||
numbers = re.findall(r"\d+", answer_str)
|
||||
if len(numbers) < 1:
|
||||
return INVALID
|
||||
try:
|
||||
return ast.literal_eval(numbers[-1])
|
||||
except SyntaxError:
|
||||
return INVALID
|
||||
|
||||
|
||||
def run_gsm8k_benchmark(base_url, num_questions=200, num_shots=5, parallel=64):
|
||||
import sglang as sgl
|
||||
from sglang.lang.backend.runtime_endpoint import RuntimeEndpoint
|
||||
|
||||
url = "https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/test.jsonl"
|
||||
data_path = download_and_cache_file(url)
|
||||
lines = list(read_jsonl(data_path))
|
||||
|
||||
few_shot_examples = get_few_shot_examples(lines, num_shots)
|
||||
questions = []
|
||||
labels = []
|
||||
for i in range(len(lines[:num_questions])):
|
||||
questions.append(get_one_example(lines, i, False))
|
||||
labels.append(get_answer_value(lines[i]["answer"]))
|
||||
arguments = [{"question": q} for q in questions]
|
||||
|
||||
@sgl.function
|
||||
def few_shot_gsm8k(s, question):
|
||||
s += few_shot_examples + question
|
||||
s += sgl.gen(
|
||||
"answer", max_tokens=512, stop=["Question", "Assistant:", "<|separator|>"]
|
||||
)
|
||||
|
||||
backend = RuntimeEndpoint(base_url)
|
||||
sgl.set_default_backend(backend)
|
||||
|
||||
tic = time.perf_counter()
|
||||
states = few_shot_gsm8k.run_batch(
|
||||
arguments, temperature=0, num_threads=parallel, progress_bar=True
|
||||
)
|
||||
latency = time.perf_counter() - tic
|
||||
|
||||
preds = [get_answer_value(states[i]["answer"]) for i in range(len(states))]
|
||||
acc = np.mean(np.array(preds) == np.array(labels))
|
||||
return float(acc), float(latency)
|
||||
|
||||
|
||||
class TestDeepSeekV31EvalAMD(unittest.TestCase):
|
||||
"""DeepSeek-V3.1 GSM8K Completion Evaluation Test for AMD MI300X."""
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.base_url = DEFAULT_URL_FOR_TEST
|
||||
cls.num_questions = int(os.environ.get("GSM8K_NUM_QUESTIONS", "200"))
|
||||
cls.accuracy_threshold = 0.90
|
||||
|
||||
def test_deepseek_v31_accuracy(self):
|
||||
"""Test DeepSeek-V3.1 with GSM8K completion benchmark."""
|
||||
env = os.environ.copy()
|
||||
env["SGLANG_USE_AITER"] = "1"
|
||||
|
||||
other_args = [
|
||||
"--tp",
|
||||
"8",
|
||||
"--attention-backend",
|
||||
"aiter",
|
||||
"--chunked-prefill-size",
|
||||
"131072",
|
||||
"--mem-fraction-static",
|
||||
"0.85",
|
||||
"--trust-remote-code",
|
||||
]
|
||||
|
||||
process = popen_launch_server(
|
||||
model=DEEPSEEK_V31_MODEL_PATH,
|
||||
base_url=self.base_url,
|
||||
timeout=3600,
|
||||
other_args=other_args,
|
||||
env=env,
|
||||
)
|
||||
|
||||
try:
|
||||
acc, latency = run_gsm8k_benchmark(
|
||||
self.base_url, num_questions=self.num_questions
|
||||
)
|
||||
passed = acc >= self.accuracy_threshold
|
||||
status = "✅ PASS" if passed else "❌ FAIL"
|
||||
|
||||
summary = f"### DeepSeek-V3.1 (MI300X)\n\n"
|
||||
summary += f"| Model | Accuracy | Threshold | Status |\n"
|
||||
summary += f"| ----- | -------- | --------- | ------ |\n"
|
||||
summary += f"| {DEEPSEEK_V31_MODEL_PATH} | {acc:.3f} | {self.accuracy_threshold} | {status} |\n"
|
||||
|
||||
if is_in_ci():
|
||||
write_github_step_summary(summary)
|
||||
|
||||
self.assertGreaterEqual(
|
||||
acc,
|
||||
self.accuracy_threshold,
|
||||
f"Accuracy {acc:.3f} below threshold {self.accuracy_threshold}",
|
||||
)
|
||||
finally:
|
||||
kill_process_tree(process.pid)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
247
test/registered/amd/accuracy/test_gpt_oss_eval_amd.py
Normal file
247
test/registered/amd/accuracy/test_gpt_oss_eval_amd.py
Normal file
@@ -0,0 +1,247 @@
|
||||
"""AMD GPT-OSS GSM8K Completion Evaluation Test (8-GPU)
|
||||
|
||||
Tests GPT-OSS models (lmsys/gpt-oss-20b-bf16, lmsys/gpt-oss-120b-bf16) using
|
||||
few-shot completion benchmark on MI300X.
|
||||
|
||||
Registry: nightly-amd-8-gpu suite
|
||||
"""
|
||||
|
||||
import ast
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
import unittest
|
||||
from dataclasses import dataclass
|
||||
from typing import List, Optional, Tuple
|
||||
|
||||
import numpy as np
|
||||
|
||||
from sglang.srt.utils import kill_process_tree
|
||||
from sglang.test.ci.ci_register import register_amd_ci
|
||||
from sglang.test.test_utils import (
|
||||
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||
DEFAULT_URL_FOR_TEST,
|
||||
is_in_ci,
|
||||
popen_launch_server,
|
||||
write_github_step_summary,
|
||||
)
|
||||
from sglang.utils import download_and_cache_file, read_jsonl
|
||||
|
||||
# Register for AMD CI - GPT-OSS accuracy tests (~30 min)
|
||||
register_amd_ci(est_time=1800, suite="nightly-amd-accuracy-8-gpu-gpt-oss", nightly=True)
|
||||
|
||||
INVALID = -9999999
|
||||
|
||||
|
||||
@dataclass
|
||||
class ModelConfig:
|
||||
"""Configuration for a model to test."""
|
||||
|
||||
model_path: str
|
||||
tp_size: int = 8
|
||||
accuracy_threshold: float = 0.50
|
||||
other_args: Optional[List[str]] = None
|
||||
env_vars: Optional[dict] = None
|
||||
timeout: Optional[int] = None
|
||||
|
||||
def __post_init__(self):
|
||||
if self.other_args is None:
|
||||
self.other_args = []
|
||||
if self.env_vars is None:
|
||||
self.env_vars = {}
|
||||
|
||||
|
||||
# GPT-OSS models for MI300X
|
||||
GPT_OSS_MODELS = [
|
||||
ModelConfig(
|
||||
model_path="lmsys/gpt-oss-20b-bf16",
|
||||
tp_size=8,
|
||||
accuracy_threshold=0.47,
|
||||
other_args=[
|
||||
"--chunked-prefill-size",
|
||||
"130172",
|
||||
"--max-running-requests",
|
||||
"128",
|
||||
"--mem-fraction-static",
|
||||
"0.85",
|
||||
"--attention-backend",
|
||||
"triton",
|
||||
"--trust-remote-code",
|
||||
],
|
||||
env_vars={"SGLANG_USE_AITER": "0"},
|
||||
),
|
||||
ModelConfig(
|
||||
model_path="lmsys/gpt-oss-120b-bf16",
|
||||
tp_size=8,
|
||||
accuracy_threshold=0.79,
|
||||
timeout=900,
|
||||
other_args=[
|
||||
"--chunked-prefill-size",
|
||||
"130172",
|
||||
"--max-running-requests",
|
||||
"128",
|
||||
"--mem-fraction-static",
|
||||
"0.85",
|
||||
"--attention-backend",
|
||||
"triton",
|
||||
"--trust-remote-code",
|
||||
],
|
||||
env_vars={"SGLANG_USE_AITER": "0"},
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def get_one_example(lines, i, include_answer):
|
||||
"""Format a single GSM8K example."""
|
||||
ret = "Question: " + lines[i]["question"] + "\nAnswer:"
|
||||
if include_answer:
|
||||
ret += " " + lines[i]["answer"]
|
||||
return ret
|
||||
|
||||
|
||||
def get_few_shot_examples(lines, k):
|
||||
"""Get k few-shot examples for prompting."""
|
||||
ret = ""
|
||||
for i in range(k):
|
||||
ret += get_one_example(lines, i, True) + "\n\n"
|
||||
return ret
|
||||
|
||||
|
||||
def get_answer_value(answer_str):
|
||||
"""Extract numerical answer from response."""
|
||||
answer_str = answer_str.replace(",", "")
|
||||
numbers = re.findall(r"\d+", answer_str)
|
||||
if len(numbers) < 1:
|
||||
return INVALID
|
||||
try:
|
||||
return ast.literal_eval(numbers[-1])
|
||||
except SyntaxError:
|
||||
return INVALID
|
||||
|
||||
|
||||
def run_gsm8k_benchmark(
|
||||
base_url: str,
|
||||
num_questions: int = 200,
|
||||
num_shots: int = 5,
|
||||
parallel: int = 64,
|
||||
) -> Tuple[float, float, float]:
|
||||
"""Run GSM8K few-shot completion benchmark."""
|
||||
import sglang as sgl
|
||||
from sglang.lang.backend.runtime_endpoint import RuntimeEndpoint
|
||||
|
||||
url = "https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/test.jsonl"
|
||||
data_path = download_and_cache_file(url)
|
||||
lines = list(read_jsonl(data_path))
|
||||
|
||||
few_shot_examples = get_few_shot_examples(lines, num_shots)
|
||||
|
||||
questions = []
|
||||
labels = []
|
||||
for i in range(len(lines[:num_questions])):
|
||||
questions.append(get_one_example(lines, i, False))
|
||||
labels.append(get_answer_value(lines[i]["answer"]))
|
||||
assert all(l != INVALID for l in labels)
|
||||
arguments = [{"question": q} for q in questions]
|
||||
|
||||
@sgl.function
|
||||
def few_shot_gsm8k(s, question):
|
||||
s += few_shot_examples + question
|
||||
s += sgl.gen(
|
||||
"answer", max_tokens=512, stop=["Question", "Assistant:", "<|separator|>"]
|
||||
)
|
||||
|
||||
backend = RuntimeEndpoint(base_url)
|
||||
sgl.set_default_backend(backend)
|
||||
|
||||
tic = time.perf_counter()
|
||||
states = few_shot_gsm8k.run_batch(
|
||||
arguments, temperature=0, num_threads=parallel, progress_bar=True
|
||||
)
|
||||
latency = time.perf_counter() - tic
|
||||
|
||||
preds = [get_answer_value(states[i]["answer"]) for i in range(len(states))]
|
||||
acc = np.mean(np.array(preds) == np.array(labels))
|
||||
invalid = np.mean(np.array(preds) == INVALID)
|
||||
|
||||
return float(acc), float(invalid), float(latency)
|
||||
|
||||
|
||||
class TestGptOssEvalAMD(unittest.TestCase):
|
||||
"""GPT-OSS GSM8K Completion Evaluation Test for AMD MI300X."""
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.models = GPT_OSS_MODELS
|
||||
cls.base_url = DEFAULT_URL_FOR_TEST
|
||||
cls.num_questions = int(os.environ.get("GSM8K_NUM_QUESTIONS", "200"))
|
||||
|
||||
def test_gpt_oss_accuracy(self):
|
||||
"""Test GPT-OSS models with GSM8K completion benchmark."""
|
||||
all_results = []
|
||||
summary = "### GPT-OSS Models (MI300X)\n\n"
|
||||
summary += "| Model | TP | Accuracy | Threshold | Status |\n"
|
||||
summary += "| ----- | -- | -------- | --------- | ------ |\n"
|
||||
|
||||
for config in self.models:
|
||||
with self.subTest(model=config.model_path):
|
||||
print(f"\n{'='*60}")
|
||||
print(f"Testing: {config.model_path}")
|
||||
print(f"{'='*60}")
|
||||
|
||||
env = os.environ.copy()
|
||||
for key, value in config.env_vars.items():
|
||||
env[key] = value
|
||||
|
||||
other_args = list(config.other_args)
|
||||
other_args.extend(["--tp", str(config.tp_size)])
|
||||
timeout = config.timeout or DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH
|
||||
|
||||
try:
|
||||
process = popen_launch_server(
|
||||
model=config.model_path,
|
||||
base_url=self.base_url,
|
||||
timeout=timeout,
|
||||
other_args=other_args,
|
||||
env=env,
|
||||
)
|
||||
|
||||
try:
|
||||
acc, invalid, latency = run_gsm8k_benchmark(
|
||||
self.base_url, num_questions=self.num_questions
|
||||
)
|
||||
passed = acc >= config.accuracy_threshold
|
||||
status = "✅ PASS" if passed else "❌ FAIL"
|
||||
|
||||
all_results.append(
|
||||
{
|
||||
"model": config.model_path,
|
||||
"accuracy": acc,
|
||||
"passed": passed,
|
||||
}
|
||||
)
|
||||
summary += f"| {config.model_path} | {config.tp_size} | {acc:.3f} | {config.accuracy_threshold} | {status} |\n"
|
||||
|
||||
finally:
|
||||
kill_process_tree(process.pid)
|
||||
|
||||
except Exception as e:
|
||||
summary += f"| {config.model_path} | {config.tp_size} | N/A | {config.accuracy_threshold} | ❌ ERROR |\n"
|
||||
all_results.append(
|
||||
{
|
||||
"model": config.model_path,
|
||||
"accuracy": None,
|
||||
"passed": False,
|
||||
"error": str(e),
|
||||
}
|
||||
)
|
||||
|
||||
if is_in_ci():
|
||||
write_github_step_summary(summary)
|
||||
|
||||
failed = [r for r in all_results if not r["passed"]]
|
||||
if failed:
|
||||
raise AssertionError(f"Failed models: {[r['model'] for r in failed]}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
162
test/registered/amd/accuracy/test_grok1_fp8_eval_amd.py
Normal file
162
test/registered/amd/accuracy/test_grok1_fp8_eval_amd.py
Normal file
@@ -0,0 +1,162 @@
|
||||
"""AMD GROK1-FP8 GSM8K Completion Evaluation Test (8-GPU)
|
||||
|
||||
Tests Grok-1 FP8 model using few-shot completion benchmark on MI300X.
|
||||
|
||||
Registry: nightly-amd-8-gpu-grok1-fp8 suite
|
||||
"""
|
||||
|
||||
import ast
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
import unittest
|
||||
|
||||
import numpy as np
|
||||
|
||||
from sglang.srt.utils import kill_process_tree
|
||||
from sglang.test.ci.ci_register import register_amd_ci
|
||||
from sglang.test.test_utils import (
|
||||
DEFAULT_URL_FOR_TEST,
|
||||
is_in_ci,
|
||||
popen_launch_server,
|
||||
write_github_step_summary,
|
||||
)
|
||||
from sglang.utils import download_and_cache_file, read_jsonl
|
||||
|
||||
# Register for AMD CI - GROK1-FP8 accuracy tests (~25 min)
|
||||
register_amd_ci(
|
||||
est_time=1500, suite="nightly-amd-accuracy-8-gpu-grok1-fp8", nightly=True
|
||||
)
|
||||
|
||||
INVALID = -9999999
|
||||
|
||||
GROK1_FP8_MODEL_PATH = os.environ.get("GROK1_MODEL_PATH", "lmzheng/grok-1")
|
||||
GROK1_TOKENIZER_PATH = os.environ.get("GROK1_TOKENIZER_PATH", "Xenova/grok-1-tokenizer")
|
||||
|
||||
|
||||
def get_one_example(lines, i, include_answer):
|
||||
ret = "Question: " + lines[i]["question"] + "\nAnswer:"
|
||||
if include_answer:
|
||||
ret += " " + lines[i]["answer"]
|
||||
return ret
|
||||
|
||||
|
||||
def get_few_shot_examples(lines, k):
|
||||
ret = ""
|
||||
for i in range(k):
|
||||
ret += get_one_example(lines, i, True) + "\n\n"
|
||||
return ret
|
||||
|
||||
|
||||
def get_answer_value(answer_str):
|
||||
answer_str = answer_str.replace(",", "")
|
||||
numbers = re.findall(r"\d+", answer_str)
|
||||
if len(numbers) < 1:
|
||||
return INVALID
|
||||
try:
|
||||
return ast.literal_eval(numbers[-1])
|
||||
except SyntaxError:
|
||||
return INVALID
|
||||
|
||||
|
||||
def run_gsm8k_benchmark(base_url, num_questions=200, num_shots=5, parallel=64):
|
||||
import sglang as sgl
|
||||
from sglang.lang.backend.runtime_endpoint import RuntimeEndpoint
|
||||
|
||||
url = "https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/test.jsonl"
|
||||
data_path = download_and_cache_file(url)
|
||||
lines = list(read_jsonl(data_path))
|
||||
|
||||
few_shot_examples = get_few_shot_examples(lines, num_shots)
|
||||
questions = []
|
||||
labels = []
|
||||
for i in range(len(lines[:num_questions])):
|
||||
questions.append(get_one_example(lines, i, False))
|
||||
labels.append(get_answer_value(lines[i]["answer"]))
|
||||
arguments = [{"question": q} for q in questions]
|
||||
|
||||
@sgl.function
|
||||
def few_shot_gsm8k(s, question):
|
||||
s += few_shot_examples + question
|
||||
s += sgl.gen(
|
||||
"answer", max_tokens=512, stop=["Question", "Assistant:", "<|separator|>"]
|
||||
)
|
||||
|
||||
backend = RuntimeEndpoint(base_url)
|
||||
sgl.set_default_backend(backend)
|
||||
|
||||
tic = time.perf_counter()
|
||||
states = few_shot_gsm8k.run_batch(
|
||||
arguments, temperature=0, num_threads=parallel, progress_bar=True
|
||||
)
|
||||
latency = time.perf_counter() - tic
|
||||
|
||||
preds = [get_answer_value(states[i]["answer"]) for i in range(len(states))]
|
||||
acc = np.mean(np.array(preds) == np.array(labels))
|
||||
return float(acc), float(latency)
|
||||
|
||||
|
||||
class TestGrok1FP8EvalAMD(unittest.TestCase):
|
||||
"""GROK1-FP8 GSM8K Completion Evaluation Test for AMD MI300X."""
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.base_url = DEFAULT_URL_FOR_TEST
|
||||
cls.num_questions = int(os.environ.get("GSM8K_NUM_QUESTIONS", "200"))
|
||||
cls.accuracy_threshold = 0.80
|
||||
|
||||
def test_grok1_fp8_accuracy(self):
|
||||
"""Test Grok-1 FP8 with GSM8K completion benchmark."""
|
||||
env = os.environ.copy()
|
||||
env["RCCL_MSCCL_ENABLE"] = "0"
|
||||
env["SGLANG_USE_AITER"] = "1"
|
||||
env["SGLANG_INT4_WEIGHT"] = "0"
|
||||
|
||||
other_args = [
|
||||
"--tp",
|
||||
"8",
|
||||
"--quantization",
|
||||
"fp8",
|
||||
"--attention-backend",
|
||||
"aiter",
|
||||
"--mem-fraction-static",
|
||||
"0.85",
|
||||
"--tokenizer-path",
|
||||
GROK1_TOKENIZER_PATH,
|
||||
"--trust-remote-code",
|
||||
]
|
||||
|
||||
process = popen_launch_server(
|
||||
model=GROK1_FP8_MODEL_PATH,
|
||||
base_url=self.base_url,
|
||||
timeout=3600,
|
||||
other_args=other_args,
|
||||
env=env,
|
||||
)
|
||||
|
||||
try:
|
||||
acc, latency = run_gsm8k_benchmark(
|
||||
self.base_url, num_questions=self.num_questions
|
||||
)
|
||||
passed = acc >= self.accuracy_threshold
|
||||
status = "✅ PASS" if passed else "❌ FAIL"
|
||||
|
||||
summary = f"### GROK1-FP8 (MI300X)\n\n"
|
||||
summary += f"| Model | Accuracy | Threshold | Status |\n"
|
||||
summary += f"| ----- | -------- | --------- | ------ |\n"
|
||||
summary += f"| {GROK1_FP8_MODEL_PATH} | {acc:.3f} | {self.accuracy_threshold} | {status} |\n"
|
||||
|
||||
if is_in_ci():
|
||||
write_github_step_summary(summary)
|
||||
|
||||
self.assertGreaterEqual(
|
||||
acc,
|
||||
self.accuracy_threshold,
|
||||
f"Accuracy {acc:.3f} below threshold {self.accuracy_threshold}",
|
||||
)
|
||||
finally:
|
||||
kill_process_tree(process.pid)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
162
test/registered/amd/accuracy/test_grok1_int4_eval_amd.py
Normal file
162
test/registered/amd/accuracy/test_grok1_int4_eval_amd.py
Normal file
@@ -0,0 +1,162 @@
|
||||
"""AMD GROK1-INT4 GSM8K Completion Evaluation Test (8-GPU)
|
||||
|
||||
Tests Grok-1 INT4 (W4A8KV8) model using few-shot completion benchmark on MI300X.
|
||||
|
||||
Registry: nightly-amd-8-gpu-grok1-int4 suite
|
||||
"""
|
||||
|
||||
import ast
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
import unittest
|
||||
|
||||
import numpy as np
|
||||
|
||||
from sglang.srt.utils import kill_process_tree
|
||||
from sglang.test.ci.ci_register import register_amd_ci
|
||||
from sglang.test.test_utils import (
|
||||
DEFAULT_URL_FOR_TEST,
|
||||
is_in_ci,
|
||||
popen_launch_server,
|
||||
write_github_step_summary,
|
||||
)
|
||||
from sglang.utils import download_and_cache_file, read_jsonl
|
||||
|
||||
# Register for AMD CI - GROK1-INT4 accuracy tests (~25 min)
|
||||
register_amd_ci(
|
||||
est_time=1500, suite="nightly-amd-accuracy-8-gpu-grok1-int4", nightly=True
|
||||
)
|
||||
|
||||
INVALID = -9999999
|
||||
|
||||
GROK1_INT4_MODEL_PATH = os.environ.get("GROK1_INT4_MODEL_PATH", "amd/grok-1-W4A8KV8")
|
||||
GROK1_TOKENIZER_PATH = os.environ.get("GROK1_TOKENIZER_PATH", "Xenova/grok-1-tokenizer")
|
||||
|
||||
|
||||
def get_one_example(lines, i, include_answer):
|
||||
ret = "Question: " + lines[i]["question"] + "\nAnswer:"
|
||||
if include_answer:
|
||||
ret += " " + lines[i]["answer"]
|
||||
return ret
|
||||
|
||||
|
||||
def get_few_shot_examples(lines, k):
|
||||
ret = ""
|
||||
for i in range(k):
|
||||
ret += get_one_example(lines, i, True) + "\n\n"
|
||||
return ret
|
||||
|
||||
|
||||
def get_answer_value(answer_str):
|
||||
answer_str = answer_str.replace(",", "")
|
||||
numbers = re.findall(r"\d+", answer_str)
|
||||
if len(numbers) < 1:
|
||||
return INVALID
|
||||
try:
|
||||
return ast.literal_eval(numbers[-1])
|
||||
except SyntaxError:
|
||||
return INVALID
|
||||
|
||||
|
||||
def run_gsm8k_benchmark(base_url, num_questions=200, num_shots=5, parallel=64):
|
||||
import sglang as sgl
|
||||
from sglang.lang.backend.runtime_endpoint import RuntimeEndpoint
|
||||
|
||||
url = "https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/test.jsonl"
|
||||
data_path = download_and_cache_file(url)
|
||||
lines = list(read_jsonl(data_path))
|
||||
|
||||
few_shot_examples = get_few_shot_examples(lines, num_shots)
|
||||
questions = []
|
||||
labels = []
|
||||
for i in range(len(lines[:num_questions])):
|
||||
questions.append(get_one_example(lines, i, False))
|
||||
labels.append(get_answer_value(lines[i]["answer"]))
|
||||
arguments = [{"question": q} for q in questions]
|
||||
|
||||
@sgl.function
|
||||
def few_shot_gsm8k(s, question):
|
||||
s += few_shot_examples + question
|
||||
s += sgl.gen(
|
||||
"answer", max_tokens=512, stop=["Question", "Assistant:", "<|separator|>"]
|
||||
)
|
||||
|
||||
backend = RuntimeEndpoint(base_url)
|
||||
sgl.set_default_backend(backend)
|
||||
|
||||
tic = time.perf_counter()
|
||||
states = few_shot_gsm8k.run_batch(
|
||||
arguments, temperature=0, num_threads=parallel, progress_bar=True
|
||||
)
|
||||
latency = time.perf_counter() - tic
|
||||
|
||||
preds = [get_answer_value(states[i]["answer"]) for i in range(len(states))]
|
||||
acc = np.mean(np.array(preds) == np.array(labels))
|
||||
return float(acc), float(latency)
|
||||
|
||||
|
||||
class TestGrok1INT4EvalAMD(unittest.TestCase):
|
||||
"""GROK1-INT4 GSM8K Completion Evaluation Test for AMD MI300X."""
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.base_url = DEFAULT_URL_FOR_TEST
|
||||
cls.num_questions = int(os.environ.get("GSM8K_NUM_QUESTIONS", "200"))
|
||||
cls.accuracy_threshold = 0.80
|
||||
|
||||
def test_grok1_int4_accuracy(self):
|
||||
"""Test Grok-1 INT4 with GSM8K completion benchmark."""
|
||||
env = os.environ.copy()
|
||||
env["RCCL_MSCCL_ENABLE"] = "0"
|
||||
env["SGLANG_USE_AITER"] = "1"
|
||||
env["SGLANG_INT4_WEIGHT"] = "1"
|
||||
|
||||
other_args = [
|
||||
"--tp",
|
||||
"8",
|
||||
"--quantization",
|
||||
"fp8",
|
||||
"--attention-backend",
|
||||
"aiter",
|
||||
"--mem-fraction-static",
|
||||
"0.85",
|
||||
"--tokenizer-path",
|
||||
GROK1_TOKENIZER_PATH,
|
||||
"--trust-remote-code",
|
||||
]
|
||||
|
||||
process = popen_launch_server(
|
||||
model=GROK1_INT4_MODEL_PATH,
|
||||
base_url=self.base_url,
|
||||
timeout=3600,
|
||||
other_args=other_args,
|
||||
env=env,
|
||||
)
|
||||
|
||||
try:
|
||||
acc, latency = run_gsm8k_benchmark(
|
||||
self.base_url, num_questions=self.num_questions
|
||||
)
|
||||
passed = acc >= self.accuracy_threshold
|
||||
status = "✅ PASS" if passed else "❌ FAIL"
|
||||
|
||||
summary = f"### GROK1-INT4 (MI300X)\n\n"
|
||||
summary += f"| Model | Accuracy | Threshold | Status |\n"
|
||||
summary += f"| ----- | -------- | --------- | ------ |\n"
|
||||
summary += f"| {GROK1_INT4_MODEL_PATH} | {acc:.3f} | {self.accuracy_threshold} | {status} |\n"
|
||||
|
||||
if is_in_ci():
|
||||
write_github_step_summary(summary)
|
||||
|
||||
self.assertGreaterEqual(
|
||||
acc,
|
||||
self.accuracy_threshold,
|
||||
f"Accuracy {acc:.3f} below threshold {self.accuracy_threshold}",
|
||||
)
|
||||
finally:
|
||||
kill_process_tree(process.pid)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
162
test/registered/amd/accuracy/test_grok2_eval_amd.py
Normal file
162
test/registered/amd/accuracy/test_grok2_eval_amd.py
Normal file
@@ -0,0 +1,162 @@
|
||||
"""AMD GROK2 GSM8K Completion Evaluation Test (8-GPU)
|
||||
|
||||
Tests Grok-2 model using few-shot completion benchmark on MI300X.
|
||||
|
||||
Registry: nightly-amd-8-gpu-grok2 suite
|
||||
"""
|
||||
|
||||
import ast
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
import unittest
|
||||
|
||||
import numpy as np
|
||||
|
||||
from sglang.srt.utils import kill_process_tree
|
||||
from sglang.test.ci.ci_register import register_amd_ci
|
||||
from sglang.test.test_utils import (
|
||||
DEFAULT_URL_FOR_TEST,
|
||||
is_in_ci,
|
||||
popen_launch_server,
|
||||
write_github_step_summary,
|
||||
)
|
||||
from sglang.utils import download_and_cache_file, read_jsonl
|
||||
|
||||
# Register for AMD CI - GROK2 accuracy tests (~25 min)
|
||||
register_amd_ci(est_time=1500, suite="nightly-amd-accuracy-8-gpu-grok2", nightly=True)
|
||||
|
||||
INVALID = -9999999
|
||||
|
||||
GROK2_MODEL_PATH = os.environ.get("GROK2_MODEL_PATH", "xai-org/grok-2")
|
||||
GROK2_TOKENIZER_PATH = os.environ.get(
|
||||
"GROK2_TOKENIZER_PATH", "alvarobartt/grok-2-tokenizer"
|
||||
)
|
||||
|
||||
|
||||
def get_one_example(lines, i, include_answer):
|
||||
ret = "Question: " + lines[i]["question"] + "\nAnswer:"
|
||||
if include_answer:
|
||||
ret += " " + lines[i]["answer"]
|
||||
return ret
|
||||
|
||||
|
||||
def get_few_shot_examples(lines, k):
|
||||
ret = ""
|
||||
for i in range(k):
|
||||
ret += get_one_example(lines, i, True) + "\n\n"
|
||||
return ret
|
||||
|
||||
|
||||
def get_answer_value(answer_str):
|
||||
answer_str = answer_str.replace(",", "")
|
||||
numbers = re.findall(r"\d+", answer_str)
|
||||
if len(numbers) < 1:
|
||||
return INVALID
|
||||
try:
|
||||
return ast.literal_eval(numbers[-1])
|
||||
except SyntaxError:
|
||||
return INVALID
|
||||
|
||||
|
||||
def run_gsm8k_benchmark(base_url, num_questions=200, num_shots=5, parallel=64):
|
||||
import sglang as sgl
|
||||
from sglang.lang.backend.runtime_endpoint import RuntimeEndpoint
|
||||
|
||||
url = "https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/test.jsonl"
|
||||
data_path = download_and_cache_file(url)
|
||||
lines = list(read_jsonl(data_path))
|
||||
|
||||
few_shot_examples = get_few_shot_examples(lines, num_shots)
|
||||
questions = []
|
||||
labels = []
|
||||
for i in range(len(lines[:num_questions])):
|
||||
questions.append(get_one_example(lines, i, False))
|
||||
labels.append(get_answer_value(lines[i]["answer"]))
|
||||
arguments = [{"question": q} for q in questions]
|
||||
|
||||
@sgl.function
|
||||
def few_shot_gsm8k(s, question):
|
||||
s += few_shot_examples + question
|
||||
s += sgl.gen(
|
||||
"answer", max_tokens=512, stop=["Question", "Assistant:", "<|separator|>"]
|
||||
)
|
||||
|
||||
backend = RuntimeEndpoint(base_url)
|
||||
sgl.set_default_backend(backend)
|
||||
|
||||
tic = time.perf_counter()
|
||||
states = few_shot_gsm8k.run_batch(
|
||||
arguments, temperature=0, num_threads=parallel, progress_bar=True
|
||||
)
|
||||
latency = time.perf_counter() - tic
|
||||
|
||||
preds = [get_answer_value(states[i]["answer"]) for i in range(len(states))]
|
||||
acc = np.mean(np.array(preds) == np.array(labels))
|
||||
return float(acc), float(latency)
|
||||
|
||||
|
||||
class TestGrok2EvalAMD(unittest.TestCase):
|
||||
"""GROK2 GSM8K Completion Evaluation Test for AMD MI300X."""
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.base_url = DEFAULT_URL_FOR_TEST
|
||||
cls.num_questions = int(os.environ.get("GSM8K_NUM_QUESTIONS", "200"))
|
||||
cls.accuracy_threshold = 0.915
|
||||
|
||||
def test_grok2_accuracy(self):
|
||||
"""Test Grok-2 with GSM8K completion benchmark."""
|
||||
env = os.environ.copy()
|
||||
env["RCCL_MSCCL_ENABLE"] = "0"
|
||||
env["SGLANG_USE_AITER"] = "1"
|
||||
env["SGLANG_INT4_WEIGHT"] = "0"
|
||||
|
||||
other_args = [
|
||||
"--tp",
|
||||
"8",
|
||||
"--quantization",
|
||||
"fp8",
|
||||
"--attention-backend",
|
||||
"aiter",
|
||||
"--mem-fraction-static",
|
||||
"0.85",
|
||||
"--tokenizer-path",
|
||||
GROK2_TOKENIZER_PATH,
|
||||
"--trust-remote-code",
|
||||
]
|
||||
|
||||
process = popen_launch_server(
|
||||
model=GROK2_MODEL_PATH,
|
||||
base_url=self.base_url,
|
||||
timeout=3600,
|
||||
other_args=other_args,
|
||||
env=env,
|
||||
)
|
||||
|
||||
try:
|
||||
acc, latency = run_gsm8k_benchmark(
|
||||
self.base_url, num_questions=self.num_questions
|
||||
)
|
||||
passed = acc >= self.accuracy_threshold
|
||||
status = "✅ PASS" if passed else "❌ FAIL"
|
||||
|
||||
summary = f"### GROK2 (MI300X)\n\n"
|
||||
summary += f"| Model | Accuracy | Threshold | Status |\n"
|
||||
summary += f"| ----- | -------- | --------- | ------ |\n"
|
||||
summary += f"| {GROK2_MODEL_PATH} | {acc:.3f} | {self.accuracy_threshold} | {status} |\n"
|
||||
|
||||
if is_in_ci():
|
||||
write_github_step_summary(summary)
|
||||
|
||||
self.assertGreaterEqual(
|
||||
acc,
|
||||
self.accuracy_threshold,
|
||||
f"Accuracy {acc:.3f} below threshold {self.accuracy_threshold}",
|
||||
)
|
||||
finally:
|
||||
kill_process_tree(process.pid)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
287
test/registered/amd/accuracy/test_grok_eval_amd.py
Normal file
287
test/registered/amd/accuracy/test_grok_eval_amd.py
Normal file
@@ -0,0 +1,287 @@
|
||||
"""AMD GROK GSM8K Completion Evaluation Test (8-GPU)
|
||||
|
||||
Tests GROK models (Grok-1 FP8, Grok-1 INT4, Grok-2) using
|
||||
few-shot completion benchmark on MI300X.
|
||||
|
||||
Registry: nightly-amd-8-gpu-grok suite
|
||||
"""
|
||||
|
||||
import ast
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
import unittest
|
||||
from dataclasses import dataclass
|
||||
from typing import List, Optional, Tuple
|
||||
|
||||
import numpy as np
|
||||
|
||||
from sglang.srt.utils import kill_process_tree
|
||||
from sglang.test.ci.ci_register import register_amd_ci
|
||||
from sglang.test.test_utils import (
|
||||
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||
DEFAULT_URL_FOR_TEST,
|
||||
is_in_ci,
|
||||
popen_launch_server,
|
||||
write_github_step_summary,
|
||||
)
|
||||
from sglang.utils import download_and_cache_file, read_jsonl
|
||||
|
||||
# DISABLED: Split into individual files for each model variant
|
||||
# See: test_grok1_fp8_eval_amd.py, test_grok1_int4_eval_amd.py, test_grok2_eval_amd.py
|
||||
register_amd_ci(
|
||||
est_time=2700,
|
||||
suite="nightly-amd-8-gpu-grok",
|
||||
nightly=True,
|
||||
disabled="Split into test_grok1_fp8_eval_amd.py, test_grok1_int4_eval_amd.py, test_grok2_eval_amd.py",
|
||||
)
|
||||
|
||||
INVALID = -9999999
|
||||
|
||||
|
||||
@dataclass
|
||||
class ModelConfig:
|
||||
"""Configuration for a model to test."""
|
||||
|
||||
model_path: str
|
||||
tp_size: int = 8
|
||||
accuracy_threshold: float = 0.50
|
||||
other_args: Optional[List[str]] = None
|
||||
env_vars: Optional[dict] = None
|
||||
tokenizer_path: Optional[str] = None
|
||||
timeout: Optional[int] = None
|
||||
|
||||
def __post_init__(self):
|
||||
if self.other_args is None:
|
||||
self.other_args = []
|
||||
if self.env_vars is None:
|
||||
self.env_vars = {}
|
||||
|
||||
|
||||
# GROK models for MI300X
|
||||
GROK_MODELS = [
|
||||
# GROK1-FP8
|
||||
ModelConfig(
|
||||
model_path="lmzheng/grok-1",
|
||||
tp_size=8,
|
||||
accuracy_threshold=0.80,
|
||||
timeout=3600,
|
||||
tokenizer_path="Xenova/grok-1-tokenizer",
|
||||
other_args=[
|
||||
"--quantization",
|
||||
"fp8",
|
||||
"--attention-backend",
|
||||
"aiter",
|
||||
"--mem-fraction-static",
|
||||
"0.85",
|
||||
"--trust-remote-code",
|
||||
],
|
||||
env_vars={
|
||||
"RCCL_MSCCL_ENABLE": "0",
|
||||
"SGLANG_USE_AITER": "1",
|
||||
"SGLANG_INT4_WEIGHT": "0",
|
||||
},
|
||||
),
|
||||
# GROK1-INT4
|
||||
ModelConfig(
|
||||
model_path="amd/grok-1-W4A8KV8",
|
||||
tp_size=8,
|
||||
accuracy_threshold=0.80,
|
||||
timeout=3600,
|
||||
tokenizer_path="Xenova/grok-1-tokenizer",
|
||||
other_args=[
|
||||
"--quantization",
|
||||
"fp8",
|
||||
"--attention-backend",
|
||||
"aiter",
|
||||
"--mem-fraction-static",
|
||||
"0.85",
|
||||
"--trust-remote-code",
|
||||
],
|
||||
env_vars={
|
||||
"RCCL_MSCCL_ENABLE": "0",
|
||||
"SGLANG_USE_AITER": "1",
|
||||
"SGLANG_INT4_WEIGHT": "1",
|
||||
},
|
||||
),
|
||||
# GROK2
|
||||
ModelConfig(
|
||||
model_path="xai-org/grok-2",
|
||||
tp_size=8,
|
||||
accuracy_threshold=0.915,
|
||||
timeout=3600,
|
||||
tokenizer_path="alvarobartt/grok-2-tokenizer",
|
||||
other_args=[
|
||||
"--quantization",
|
||||
"fp8",
|
||||
"--attention-backend",
|
||||
"aiter",
|
||||
"--mem-fraction-static",
|
||||
"0.85",
|
||||
"--trust-remote-code",
|
||||
],
|
||||
env_vars={
|
||||
"RCCL_MSCCL_ENABLE": "0",
|
||||
"SGLANG_USE_AITER": "1",
|
||||
"SGLANG_INT4_WEIGHT": "0",
|
||||
},
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def get_one_example(lines, i, include_answer):
|
||||
"""Format a single GSM8K example."""
|
||||
ret = "Question: " + lines[i]["question"] + "\nAnswer:"
|
||||
if include_answer:
|
||||
ret += " " + lines[i]["answer"]
|
||||
return ret
|
||||
|
||||
|
||||
def get_few_shot_examples(lines, k):
|
||||
"""Get k few-shot examples for prompting."""
|
||||
ret = ""
|
||||
for i in range(k):
|
||||
ret += get_one_example(lines, i, True) + "\n\n"
|
||||
return ret
|
||||
|
||||
|
||||
def get_answer_value(answer_str):
|
||||
"""Extract numerical answer from response."""
|
||||
answer_str = answer_str.replace(",", "")
|
||||
numbers = re.findall(r"\d+", answer_str)
|
||||
if len(numbers) < 1:
|
||||
return INVALID
|
||||
try:
|
||||
return ast.literal_eval(numbers[-1])
|
||||
except SyntaxError:
|
||||
return INVALID
|
||||
|
||||
|
||||
def run_gsm8k_benchmark(
|
||||
base_url: str,
|
||||
num_questions: int = 200,
|
||||
num_shots: int = 5,
|
||||
parallel: int = 64,
|
||||
) -> Tuple[float, float, float]:
|
||||
"""Run GSM8K few-shot completion benchmark."""
|
||||
import sglang as sgl
|
||||
from sglang.lang.backend.runtime_endpoint import RuntimeEndpoint
|
||||
|
||||
url = "https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/test.jsonl"
|
||||
data_path = download_and_cache_file(url)
|
||||
lines = list(read_jsonl(data_path))
|
||||
|
||||
few_shot_examples = get_few_shot_examples(lines, num_shots)
|
||||
|
||||
questions = []
|
||||
labels = []
|
||||
for i in range(len(lines[:num_questions])):
|
||||
questions.append(get_one_example(lines, i, False))
|
||||
labels.append(get_answer_value(lines[i]["answer"]))
|
||||
assert all(l != INVALID for l in labels)
|
||||
arguments = [{"question": q} for q in questions]
|
||||
|
||||
@sgl.function
|
||||
def few_shot_gsm8k(s, question):
|
||||
s += few_shot_examples + question
|
||||
s += sgl.gen(
|
||||
"answer", max_tokens=512, stop=["Question", "Assistant:", "<|separator|>"]
|
||||
)
|
||||
|
||||
backend = RuntimeEndpoint(base_url)
|
||||
sgl.set_default_backend(backend)
|
||||
|
||||
tic = time.perf_counter()
|
||||
states = few_shot_gsm8k.run_batch(
|
||||
arguments, temperature=0, num_threads=parallel, progress_bar=True
|
||||
)
|
||||
latency = time.perf_counter() - tic
|
||||
|
||||
preds = [get_answer_value(states[i]["answer"]) for i in range(len(states))]
|
||||
acc = np.mean(np.array(preds) == np.array(labels))
|
||||
invalid = np.mean(np.array(preds) == INVALID)
|
||||
|
||||
return float(acc), float(invalid), float(latency)
|
||||
|
||||
|
||||
class TestGrokEvalAMD(unittest.TestCase):
|
||||
"""GROK GSM8K Completion Evaluation Test for AMD MI300X."""
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.models = GROK_MODELS
|
||||
cls.base_url = DEFAULT_URL_FOR_TEST
|
||||
cls.num_questions = int(os.environ.get("GSM8K_NUM_QUESTIONS", "200"))
|
||||
|
||||
def test_grok_accuracy(self):
|
||||
"""Test GROK models with GSM8K completion benchmark."""
|
||||
all_results = []
|
||||
summary = "### GROK Models (MI300X)\n\n"
|
||||
summary += "| Model | TP | Accuracy | Threshold | Status |\n"
|
||||
summary += "| ----- | -- | -------- | --------- | ------ |\n"
|
||||
|
||||
for config in self.models:
|
||||
with self.subTest(model=config.model_path):
|
||||
print(f"\n{'='*60}")
|
||||
print(f"Testing: {config.model_path}")
|
||||
print(f"{'='*60}")
|
||||
|
||||
env = os.environ.copy()
|
||||
for key, value in config.env_vars.items():
|
||||
env[key] = value
|
||||
|
||||
other_args = list(config.other_args)
|
||||
other_args.extend(["--tp", str(config.tp_size)])
|
||||
if config.tokenizer_path:
|
||||
other_args.extend(["--tokenizer-path", config.tokenizer_path])
|
||||
timeout = config.timeout or DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH
|
||||
|
||||
try:
|
||||
process = popen_launch_server(
|
||||
model=config.model_path,
|
||||
base_url=self.base_url,
|
||||
timeout=timeout,
|
||||
other_args=other_args,
|
||||
env=env,
|
||||
)
|
||||
|
||||
try:
|
||||
acc, invalid, latency = run_gsm8k_benchmark(
|
||||
self.base_url, num_questions=self.num_questions
|
||||
)
|
||||
passed = acc >= config.accuracy_threshold
|
||||
status = "✅ PASS" if passed else "❌ FAIL"
|
||||
|
||||
all_results.append(
|
||||
{
|
||||
"model": config.model_path,
|
||||
"accuracy": acc,
|
||||
"passed": passed,
|
||||
}
|
||||
)
|
||||
summary += f"| {config.model_path} | {config.tp_size} | {acc:.3f} | {config.accuracy_threshold} | {status} |\n"
|
||||
|
||||
finally:
|
||||
kill_process_tree(process.pid)
|
||||
|
||||
except Exception as e:
|
||||
summary += f"| {config.model_path} | {config.tp_size} | N/A | {config.accuracy_threshold} | ❌ ERROR |\n"
|
||||
all_results.append(
|
||||
{
|
||||
"model": config.model_path,
|
||||
"accuracy": None,
|
||||
"passed": False,
|
||||
"error": str(e),
|
||||
}
|
||||
)
|
||||
|
||||
if is_in_ci():
|
||||
write_github_step_summary(summary)
|
||||
|
||||
failed = [r for r in all_results if not r["passed"]]
|
||||
if failed:
|
||||
raise AssertionError(f"Failed models: {[r['model'] for r in failed]}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -1,5 +1,5 @@
|
||||
"""
|
||||
AMD VLM MMMU Evaluation Test (Migrated from test/srt/nightly/)
|
||||
AMD VLM MMMU Evaluation Test - MI30x Only
|
||||
|
||||
This test evaluates Vision-Language Models (VLMs) on the MMMU benchmark on AMD GPUs.
|
||||
Models are selected based on compatibility with AMD/ROCm platform.
|
||||
@@ -11,6 +11,7 @@ VLMs tested here:
|
||||
- deepseek-vl2-small
|
||||
|
||||
Note: Some VLMs from the Nvidia test are excluded due to AMD compatibility issues.
|
||||
Note: This test runs only on MI30x runners (linux-mi325-gpu-2), not on MI35x.
|
||||
|
||||
Registry: nightly-amd-vlm suite (2-GPU VLM tests)
|
||||
"""
|
||||
@@ -34,7 +35,7 @@ from sglang.test.test_utils import (
|
||||
)
|
||||
|
||||
# Register for AMD CI - VLM MMMU evaluation tests (~120 min)
|
||||
register_amd_ci(est_time=7200, suite="nightly-amd-vlm", nightly=True)
|
||||
register_amd_ci(est_time=7200, suite="nightly-amd-accuracy-2-gpu-vlm", nightly=True)
|
||||
|
||||
# AMD-verified VLM models with conservative thresholds on 100 MMMU samples
|
||||
# Format: (model_path, tp_size, accuracy_threshold, extra_args)
|
||||
@@ -1,937 +0,0 @@
|
||||
"""
|
||||
AMD GSM8K Completion Evaluation Test (Migrated from test/srt/nightly/)
|
||||
|
||||
This test uses the completion-based gsm8k benchmark (few-shot prompting)
|
||||
which works with base models that don't have chat templates.
|
||||
|
||||
This complements test_gsm8k_eval_amd.py which uses mgsm_en (chat completions)
|
||||
for instruction-tuned models.
|
||||
|
||||
Base models tested here:
|
||||
- GPT-OSS series (lmsys/gpt-oss-20b-bf16, lmsys/gpt-oss-120b-bf16)
|
||||
- GROK series (lmzheng/grok-1, amd/grok-1-W4A8KV8, xai-org/grok-2)
|
||||
- DeepSeek series (deepseek-ai/DeepSeek-V3-0324, deepseek-ai/DeepSeek-R1-0528)
|
||||
|
||||
Model groups are selected via AMD_TEST_MODEL_GROUP environment variable:
|
||||
- "gpt-oss" (default): GPT-OSS models only (nightly-amd-8-gpu-gpt-oss)
|
||||
- "grok": All GROK models (nightly-amd-8-gpu-grok)
|
||||
- "deepseek-v3-dp": DeepSeek-V3 with DP attention (nightly-amd-8-gpu-deepseek-v3-dp)
|
||||
- "deepseek-v3-tc": DeepSeek-V3 with torch compile (nightly-amd-8-gpu-deepseek-v3-tc)
|
||||
- "deepseek-v3-mtp": DeepSeek-V3 with MTP/EAGLE (nightly-amd-8-gpu-deepseek-v3-mtp)
|
||||
- "deepseek-r1": DeepSeek-R1 reasoning model (nightly-amd-8-gpu-deepseek-r1)
|
||||
- "all": All models
|
||||
|
||||
Registry: nightly-amd-8-gpu suite (8-GPU tests)
|
||||
"""
|
||||
|
||||
import ast
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import time
|
||||
import unittest
|
||||
from dataclasses import dataclass
|
||||
from typing import List, Optional, Tuple
|
||||
|
||||
import numpy as np
|
||||
|
||||
# HuggingFace Hub for model cache checking and download progress
|
||||
try:
|
||||
from huggingface_hub import HfFileSystem, snapshot_download
|
||||
from huggingface_hub.utils import GatedRepoError, RepositoryNotFoundError
|
||||
|
||||
HF_HUB_AVAILABLE = True
|
||||
except ImportError:
|
||||
HF_HUB_AVAILABLE = False
|
||||
print("[WARNING] huggingface_hub not available - model cache checking disabled")
|
||||
|
||||
from sglang.srt.utils import kill_process_tree
|
||||
from sglang.test.ci.ci_register import register_amd_ci
|
||||
from sglang.test.test_utils import (
|
||||
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||
DEFAULT_URL_FOR_TEST,
|
||||
is_in_ci,
|
||||
popen_launch_server,
|
||||
write_github_step_summary,
|
||||
)
|
||||
from sglang.utils import download_and_cache_file, read_jsonl
|
||||
|
||||
# Register for AMD CI - GSM8K completion tests (~120 min)
|
||||
register_amd_ci(est_time=7200, suite="nightly-amd-8-gpu", nightly=True)
|
||||
|
||||
INVALID = -9999999
|
||||
|
||||
|
||||
@dataclass
|
||||
class BaseModelConfig:
|
||||
"""Configuration for a base model to test."""
|
||||
|
||||
model_path: str
|
||||
tp_size: int = 8
|
||||
accuracy_threshold: float = 0.50
|
||||
other_args: Optional[List[str]] = None
|
||||
env_vars: Optional[dict] = None
|
||||
tokenizer_path: Optional[str] = None
|
||||
timeout: Optional[int] = None # Custom timeout for server launch (seconds)
|
||||
variant: Optional[str] = (
|
||||
None # Test variant name (e.g., "basic", "MTP", "DP", "TC")
|
||||
)
|
||||
|
||||
def __post_init__(self):
|
||||
if self.other_args is None:
|
||||
self.other_args = []
|
||||
if self.env_vars is None:
|
||||
self.env_vars = {}
|
||||
|
||||
def get_display_name(self) -> str:
|
||||
"""Return display name for logs/summary (model + variant if set)."""
|
||||
if self.variant:
|
||||
return f"{self.model_path} ({self.variant})"
|
||||
return self.model_path
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# MODEL GROUPS - Each group runs on a separate 8-GPU runner
|
||||
# =============================================================================
|
||||
|
||||
# Group 1: GPT-OSS models (cached on upstream CI)
|
||||
# Runner: nightly-amd-8-gpu
|
||||
AMD_GPT_OSS_MODELS = [
|
||||
# GPT-OSS-20B - smaller model, run first for faster feedback
|
||||
BaseModelConfig(
|
||||
model_path="lmsys/gpt-oss-20b-bf16",
|
||||
tp_size=8,
|
||||
accuracy_threshold=0.47,
|
||||
other_args=[
|
||||
"--chunked-prefill-size",
|
||||
"130172",
|
||||
"--max-running-requests",
|
||||
"128",
|
||||
"--mem-fraction-static",
|
||||
"0.85",
|
||||
"--attention-backend",
|
||||
"triton",
|
||||
"--trust-remote-code",
|
||||
],
|
||||
env_vars={"SGLANG_USE_AITER": "0"},
|
||||
),
|
||||
# GPT-OSS-120B - large model, needs longer timeout
|
||||
BaseModelConfig(
|
||||
model_path="lmsys/gpt-oss-120b-bf16",
|
||||
tp_size=8,
|
||||
accuracy_threshold=0.79,
|
||||
timeout=900, # 15 minutes for 120B model
|
||||
other_args=[
|
||||
"--chunked-prefill-size",
|
||||
"130172",
|
||||
"--max-running-requests",
|
||||
"128",
|
||||
"--mem-fraction-static",
|
||||
"0.85",
|
||||
"--attention-backend",
|
||||
"triton",
|
||||
"--trust-remote-code",
|
||||
],
|
||||
env_vars={"SGLANG_USE_AITER": "0"},
|
||||
),
|
||||
]
|
||||
|
||||
# Group 2: All GROK models
|
||||
# Runner: nightly-amd-8-gpu-grok
|
||||
# Order: GROK1-FP8 -> GROK1-IN4 -> GROK2.5
|
||||
AMD_GROK_MODELS = [
|
||||
# GROK1-FP8 - verified accuracy: 0.860, runtime: ~12.5min
|
||||
BaseModelConfig(
|
||||
model_path="lmzheng/grok-1",
|
||||
tp_size=8,
|
||||
accuracy_threshold=0.80,
|
||||
timeout=3600, # 1 hour for kernel compilation
|
||||
tokenizer_path="Xenova/grok-1-tokenizer",
|
||||
other_args=[
|
||||
"--quantization",
|
||||
"fp8",
|
||||
"--attention-backend",
|
||||
"aiter",
|
||||
"--mem-fraction-static",
|
||||
"0.85",
|
||||
"--trust-remote-code",
|
||||
],
|
||||
env_vars={
|
||||
"RCCL_MSCCL_ENABLE": "0",
|
||||
"SGLANG_USE_AITER": "1",
|
||||
"SGLANG_INT4_WEIGHT": "0",
|
||||
},
|
||||
),
|
||||
# GROK1-IN4 - verified accuracy: 0.820, runtime: ~12.5min
|
||||
BaseModelConfig(
|
||||
model_path="amd/grok-1-W4A8KV8",
|
||||
tp_size=8,
|
||||
accuracy_threshold=0.80,
|
||||
timeout=3600, # 1 hour for kernel compilation
|
||||
tokenizer_path="Xenova/grok-1-tokenizer",
|
||||
other_args=[
|
||||
"--quantization",
|
||||
"fp8",
|
||||
"--attention-backend",
|
||||
"aiter",
|
||||
"--mem-fraction-static",
|
||||
"0.85",
|
||||
"--trust-remote-code",
|
||||
],
|
||||
env_vars={
|
||||
"RCCL_MSCCL_ENABLE": "0",
|
||||
"SGLANG_USE_AITER": "1",
|
||||
"SGLANG_INT4_WEIGHT": "1",
|
||||
},
|
||||
),
|
||||
# GROK2.5 - verified accuracy: 0.945, runtime: ~14.5min
|
||||
BaseModelConfig(
|
||||
model_path="xai-org/grok-2",
|
||||
tp_size=8,
|
||||
accuracy_threshold=0.915,
|
||||
timeout=3600, # 1 hour for download + kernel compilation
|
||||
tokenizer_path="alvarobartt/grok-2-tokenizer",
|
||||
other_args=[
|
||||
"--quantization",
|
||||
"fp8",
|
||||
"--attention-backend",
|
||||
"aiter",
|
||||
"--mem-fraction-static",
|
||||
"0.85",
|
||||
"--trust-remote-code",
|
||||
],
|
||||
env_vars={
|
||||
"RCCL_MSCCL_ENABLE": "0",
|
||||
"SGLANG_USE_AITER": "1",
|
||||
"SGLANG_INT4_WEIGHT": "0",
|
||||
},
|
||||
),
|
||||
]
|
||||
|
||||
# Note: DeepSeek-V3 accuracy tests removed - V3 only used for perf tests
|
||||
# See test_deepseek_v3_perf.py and test_deepseek_v31_perf.py for V3 perf tests
|
||||
|
||||
# Group 3: DeepSeek-R1 (reasoning model) - Basic + MTP combined
|
||||
# Runner: nightly-amd-8-gpu-deepseek-r1
|
||||
AMD_DEEPSEEK_R1_MODELS = [
|
||||
# DeepSeek-R1-0528 basic - reasoning model, ~80GB per GPU
|
||||
BaseModelConfig(
|
||||
model_path="deepseek-ai/DeepSeek-R1-0528",
|
||||
tp_size=8,
|
||||
accuracy_threshold=0.93,
|
||||
timeout=3600, # 1 hour for large model
|
||||
variant="basic",
|
||||
other_args=[
|
||||
"--attention-backend",
|
||||
"aiter",
|
||||
"--chunked-prefill-size",
|
||||
"131072",
|
||||
"--disable-radix-cache",
|
||||
"--mem-fraction-static",
|
||||
"0.85",
|
||||
"--trust-remote-code",
|
||||
],
|
||||
env_vars={
|
||||
"SGLANG_USE_AITER": "1",
|
||||
},
|
||||
),
|
||||
# DeepSeek-R1-0528 with MTP (EAGLE speculative decoding)
|
||||
BaseModelConfig(
|
||||
model_path="deepseek-ai/DeepSeek-R1-0528",
|
||||
tp_size=8,
|
||||
accuracy_threshold=0.93,
|
||||
timeout=3600,
|
||||
variant="MTP",
|
||||
other_args=[
|
||||
"--chunked-prefill-size",
|
||||
"131072",
|
||||
"--speculative-algorithm",
|
||||
"EAGLE",
|
||||
"--speculative-num-steps",
|
||||
"3",
|
||||
"--speculative-eagle-topk",
|
||||
"1",
|
||||
"--speculative-num-draft-tokens",
|
||||
"4",
|
||||
"--mem-fraction-static",
|
||||
"0.7",
|
||||
"--trust-remote-code",
|
||||
],
|
||||
env_vars={
|
||||
"SGLANG_USE_AITER": "1",
|
||||
},
|
||||
),
|
||||
]
|
||||
|
||||
# Group 5: DeepSeek-R1 with DP + TC combined
|
||||
# Runner: nightly-amd-8-gpu-deepseek-r1-dp-tc
|
||||
# Combines DP attention and Torch Compile tests for DeepSeek-R1
|
||||
AMD_DEEPSEEK_R1_DP_TC_MODELS = [
|
||||
# DeepSeek-R1-0528 with DP attention
|
||||
BaseModelConfig(
|
||||
model_path="deepseek-ai/DeepSeek-R1-0528",
|
||||
tp_size=8,
|
||||
accuracy_threshold=0.93,
|
||||
timeout=3600,
|
||||
variant="DP",
|
||||
other_args=[
|
||||
"--chunked-prefill-size",
|
||||
"131072",
|
||||
"--dp-size",
|
||||
"8",
|
||||
"--enable-dp-attention",
|
||||
"--mem-fraction-static",
|
||||
"0.85",
|
||||
"--trust-remote-code",
|
||||
],
|
||||
env_vars={
|
||||
"SGLANG_USE_ROCM700A": "1",
|
||||
"SGLANG_USE_AITER": "1",
|
||||
},
|
||||
),
|
||||
# DeepSeek-R1-0528 with torch compile
|
||||
BaseModelConfig(
|
||||
model_path="deepseek-ai/DeepSeek-R1-0528",
|
||||
tp_size=8,
|
||||
accuracy_threshold=0.93,
|
||||
timeout=7200, # 2 hours for compilation
|
||||
variant="TC",
|
||||
other_args=[
|
||||
"--chunked-prefill-size",
|
||||
"131072",
|
||||
"--mem-fraction-static",
|
||||
"0.70",
|
||||
"--cuda-graph-max-bs",
|
||||
"8",
|
||||
"--enable-torch-compile",
|
||||
"--disable-cuda-graph",
|
||||
"--trust-remote-code",
|
||||
],
|
||||
env_vars={
|
||||
"SGLANG_USE_ROCM700A": "1",
|
||||
"SGLANG_USE_AITER": "1",
|
||||
},
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def get_model_group() -> str:
|
||||
"""Get the model group to test from environment variable."""
|
||||
return os.environ.get("AMD_TEST_MODEL_GROUP", "gpt-oss")
|
||||
|
||||
|
||||
def get_models_for_group(group: str) -> List[BaseModelConfig]:
|
||||
"""Get the list of models for a given group.
|
||||
|
||||
Note: DeepSeek-V3 is only used for perf tests, not accuracy tests.
|
||||
See test_deepseek_v3_perf.py and test_deepseek_v31_perf.py.
|
||||
"""
|
||||
if group == "gpt-oss":
|
||||
return AMD_GPT_OSS_MODELS
|
||||
elif group == "grok":
|
||||
return AMD_GROK_MODELS
|
||||
elif group == "deepseek-r1":
|
||||
return AMD_DEEPSEEK_R1_MODELS
|
||||
elif group == "deepseek-r1-dp-tc":
|
||||
return AMD_DEEPSEEK_R1_DP_TC_MODELS
|
||||
elif group == "deepseek-r1-all":
|
||||
# All DeepSeek-R1 variants: basic, MTP, DP, TC
|
||||
return AMD_DEEPSEEK_R1_MODELS + AMD_DEEPSEEK_R1_DP_TC_MODELS
|
||||
elif group == "all":
|
||||
return (
|
||||
AMD_GPT_OSS_MODELS
|
||||
+ AMD_GROK_MODELS
|
||||
+ AMD_DEEPSEEK_R1_MODELS
|
||||
+ AMD_DEEPSEEK_R1_DP_TC_MODELS
|
||||
)
|
||||
else:
|
||||
print(f"[WARNING] Unknown model group '{group}', using 'gpt-oss'")
|
||||
return AMD_GPT_OSS_MODELS
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# MODEL CACHE AND DOWNLOAD UTILITIES
|
||||
# =============================================================================
|
||||
|
||||
|
||||
def check_local_cache(model_path: str) -> Tuple[bool, str]:
|
||||
"""
|
||||
Check if model is cached locally.
|
||||
|
||||
Returns:
|
||||
Tuple of (is_cached, cache_path_or_message)
|
||||
"""
|
||||
# Check common HF cache locations
|
||||
cache_dirs = [
|
||||
os.path.expanduser("~/.cache/huggingface/hub"),
|
||||
"/sgl-data/hf-cache/hub",
|
||||
"/home/runner/sgl-data/hf-cache",
|
||||
]
|
||||
|
||||
# Convert model_path to cache directory format (org--model)
|
||||
cache_name = f"models--{model_path.replace('/', '--')}"
|
||||
|
||||
for cache_dir in cache_dirs:
|
||||
cache_path = os.path.join(cache_dir, cache_name)
|
||||
if os.path.exists(cache_path):
|
||||
# Check if there are snapshots
|
||||
snapshots_dir = os.path.join(cache_path, "snapshots")
|
||||
if os.path.exists(snapshots_dir) and os.listdir(snapshots_dir):
|
||||
return True, cache_path
|
||||
|
||||
return False, f"Not found in: {', '.join(cache_dirs)}"
|
||||
|
||||
|
||||
def check_hf_repo_access(model_path: str) -> Tuple[bool, str]:
|
||||
"""
|
||||
Check if HuggingFace repository is accessible.
|
||||
|
||||
Returns:
|
||||
Tuple of (is_accessible, message)
|
||||
"""
|
||||
if not HF_HUB_AVAILABLE:
|
||||
return True, "huggingface_hub not available, skipping access check"
|
||||
|
||||
try:
|
||||
fs = HfFileSystem()
|
||||
# Try to list files in the repo
|
||||
files = fs.ls(model_path, detail=False)
|
||||
if files:
|
||||
return True, f"Repository accessible ({len(files)} files)"
|
||||
else:
|
||||
return False, "Repository exists but is empty"
|
||||
except GatedRepoError:
|
||||
return False, "GATED REPO - requires authentication/approval"
|
||||
except RepositoryNotFoundError:
|
||||
return False, "REPO NOT FOUND on HuggingFace"
|
||||
except Exception as e:
|
||||
error_msg = str(e)
|
||||
if "401" in error_msg or "unauthorized" in error_msg.lower():
|
||||
return False, f"AUTH ERROR - may need HF_TOKEN: {error_msg[:100]}"
|
||||
elif "404" in error_msg:
|
||||
return False, f"NOT FOUND: {error_msg[:100]}"
|
||||
elif "timeout" in error_msg.lower() or "connection" in error_msg.lower():
|
||||
return False, f"NETWORK ERROR: {error_msg[:100]}"
|
||||
else:
|
||||
return False, f"ERROR: {error_msg[:100]}"
|
||||
|
||||
|
||||
def log_model_status(config: BaseModelConfig) -> Tuple[bool, str]:
|
||||
"""
|
||||
Log detailed model availability status.
|
||||
|
||||
Returns:
|
||||
Tuple of (is_available, status_message)
|
||||
"""
|
||||
model_path = config.model_path
|
||||
print(f"\n📦 Checking model: {model_path}")
|
||||
print("-" * 50)
|
||||
|
||||
# Check local cache first
|
||||
is_cached, cache_msg = check_local_cache(model_path)
|
||||
if is_cached:
|
||||
print(f" ✅ LOCAL CACHE: Found at {cache_msg}")
|
||||
return True, f"Cached locally at {cache_msg}"
|
||||
else:
|
||||
print(f" ⚠️ LOCAL CACHE: {cache_msg}")
|
||||
|
||||
# Check HF repo access
|
||||
is_accessible, access_msg = check_hf_repo_access(model_path)
|
||||
if is_accessible:
|
||||
print(f" ✅ HF ACCESS: {access_msg}")
|
||||
print(f" 📥 Model will be downloaded from HuggingFace (this may take a while)")
|
||||
return True, f"Will download from HF: {access_msg}"
|
||||
else:
|
||||
print(f" ❌ HF ACCESS: {access_msg}")
|
||||
return False, access_msg
|
||||
|
||||
# Also check tokenizer if specified
|
||||
if config.tokenizer_path:
|
||||
tok_cached, tok_msg = check_local_cache(config.tokenizer_path)
|
||||
if tok_cached:
|
||||
print(f" ✅ TOKENIZER CACHE: Found at {tok_msg}")
|
||||
else:
|
||||
tok_accessible, tok_access_msg = check_hf_repo_access(config.tokenizer_path)
|
||||
if tok_accessible:
|
||||
print(f" ✅ TOKENIZER HF: {tok_access_msg}")
|
||||
else:
|
||||
print(f" ⚠️ TOKENIZER: {tok_access_msg}")
|
||||
|
||||
return is_accessible, access_msg
|
||||
|
||||
|
||||
def download_model_with_progress(
|
||||
model_path: str, timeout: int = 3600
|
||||
) -> Tuple[bool, str]:
|
||||
"""
|
||||
Download model with progress logging.
|
||||
|
||||
Returns:
|
||||
Tuple of (success, message)
|
||||
"""
|
||||
if not HF_HUB_AVAILABLE:
|
||||
return True, "huggingface_hub not available, skipping pre-download"
|
||||
|
||||
print(f"\n📥 Pre-downloading model: {model_path}")
|
||||
print(f" Timeout: {timeout}s ({timeout/60:.0f} minutes)")
|
||||
print("-" * 50)
|
||||
|
||||
start_time = time.time()
|
||||
|
||||
try:
|
||||
# Use snapshot_download which shows progress
|
||||
local_dir = snapshot_download(
|
||||
repo_id=model_path,
|
||||
local_files_only=False,
|
||||
resume_download=True,
|
||||
)
|
||||
elapsed = time.time() - start_time
|
||||
print(f" ✅ Download complete in {elapsed:.1f}s")
|
||||
print(f" 📁 Location: {local_dir}")
|
||||
return True, f"Downloaded to {local_dir}"
|
||||
|
||||
except GatedRepoError:
|
||||
return False, "GATED REPO - requires authentication/approval"
|
||||
except RepositoryNotFoundError:
|
||||
return False, "REPO NOT FOUND on HuggingFace"
|
||||
except Exception as e:
|
||||
error_msg = str(e)
|
||||
elapsed = time.time() - start_time
|
||||
if elapsed >= timeout:
|
||||
return False, f"TIMEOUT after {elapsed:.0f}s: {error_msg[:100]}"
|
||||
elif "timeout" in error_msg.lower() or "connection" in error_msg.lower():
|
||||
return False, f"NETWORK ERROR after {elapsed:.0f}s: {error_msg[:100]}"
|
||||
else:
|
||||
return False, f"ERROR after {elapsed:.0f}s: {error_msg[:100]}"
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# BENCHMARK UTILITIES
|
||||
# =============================================================================
|
||||
|
||||
|
||||
def get_one_example(lines, i, include_answer):
|
||||
"""Format a single GSM8K example."""
|
||||
ret = "Question: " + lines[i]["question"] + "\nAnswer:"
|
||||
if include_answer:
|
||||
ret += " " + lines[i]["answer"]
|
||||
return ret
|
||||
|
||||
|
||||
def get_few_shot_examples(lines, k):
|
||||
"""Get k few-shot examples for prompting."""
|
||||
ret = ""
|
||||
for i in range(k):
|
||||
ret += get_one_example(lines, i, True) + "\n\n"
|
||||
return ret
|
||||
|
||||
|
||||
def get_answer_value(answer_str):
|
||||
"""Extract numerical answer from response."""
|
||||
answer_str = answer_str.replace(",", "")
|
||||
numbers = re.findall(r"\d+", answer_str)
|
||||
if len(numbers) < 1:
|
||||
return INVALID
|
||||
try:
|
||||
return ast.literal_eval(numbers[-1])
|
||||
except SyntaxError:
|
||||
return INVALID
|
||||
|
||||
|
||||
def run_gsm8k_benchmark(
|
||||
base_url: str,
|
||||
num_questions: int = 200,
|
||||
num_shots: int = 5,
|
||||
parallel: int = 64,
|
||||
) -> Tuple[float, float, float]:
|
||||
"""
|
||||
Run GSM8K few-shot completion benchmark.
|
||||
|
||||
Returns:
|
||||
Tuple of (accuracy, invalid_rate, latency)
|
||||
"""
|
||||
import sglang as sgl
|
||||
from sglang.lang.backend.runtime_endpoint import RuntimeEndpoint
|
||||
|
||||
# Download and load data
|
||||
url = "https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/test.jsonl"
|
||||
data_path = download_and_cache_file(url)
|
||||
lines = list(read_jsonl(data_path))
|
||||
|
||||
# Construct prompts
|
||||
few_shot_examples = get_few_shot_examples(lines, num_shots)
|
||||
|
||||
questions = []
|
||||
labels = []
|
||||
for i in range(len(lines[:num_questions])):
|
||||
questions.append(get_one_example(lines, i, False))
|
||||
labels.append(get_answer_value(lines[i]["answer"]))
|
||||
assert all(l != INVALID for l in labels)
|
||||
arguments = [{"question": q} for q in questions]
|
||||
|
||||
# Define sglang function
|
||||
@sgl.function
|
||||
def few_shot_gsm8k(s, question):
|
||||
s += few_shot_examples + question
|
||||
s += sgl.gen(
|
||||
"answer", max_tokens=512, stop=["Question", "Assistant:", "<|separator|>"]
|
||||
)
|
||||
|
||||
# Set backend
|
||||
backend = RuntimeEndpoint(base_url)
|
||||
sgl.set_default_backend(backend)
|
||||
|
||||
# Run benchmark
|
||||
tic = time.perf_counter()
|
||||
states = few_shot_gsm8k.run_batch(
|
||||
arguments,
|
||||
temperature=0,
|
||||
num_threads=parallel,
|
||||
progress_bar=True,
|
||||
)
|
||||
latency = time.perf_counter() - tic
|
||||
|
||||
# Extract predictions
|
||||
preds = []
|
||||
for i in range(len(states)):
|
||||
preds.append(get_answer_value(states[i]["answer"]))
|
||||
|
||||
# Compute metrics
|
||||
acc = np.mean(np.array(preds) == np.array(labels))
|
||||
invalid = np.mean(np.array(preds) == INVALID)
|
||||
|
||||
return float(acc), float(invalid), float(latency)
|
||||
|
||||
|
||||
def popen_launch_server_for_base_model(
|
||||
base_url: str,
|
||||
config: BaseModelConfig,
|
||||
) -> "subprocess.Popen":
|
||||
"""Launch server for a base model with appropriate configuration."""
|
||||
# Build environment - start with current env and add config-specific vars
|
||||
env = os.environ.copy()
|
||||
for key, value in config.env_vars.items():
|
||||
env[key] = value
|
||||
print(f"Setting env: {key}={value}")
|
||||
|
||||
# Build other_args
|
||||
other_args = list(config.other_args)
|
||||
other_args.extend(["--tp", str(config.tp_size)])
|
||||
other_args.extend(["--log-level-http", "warning"])
|
||||
|
||||
if config.tokenizer_path:
|
||||
other_args.extend(["--tokenizer-path", config.tokenizer_path])
|
||||
|
||||
# Use custom timeout if provided, otherwise use default
|
||||
timeout = config.timeout if config.timeout else DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH
|
||||
|
||||
process = popen_launch_server(
|
||||
model=config.model_path,
|
||||
base_url=base_url,
|
||||
timeout=timeout,
|
||||
other_args=other_args,
|
||||
env=env, # Pass environment explicitly
|
||||
)
|
||||
return process
|
||||
|
||||
|
||||
class TestNightlyGsm8kCompletionEvalAMD(unittest.TestCase):
|
||||
"""
|
||||
AMD GSM8K Completion Evaluation Test
|
||||
|
||||
Tests base models using few-shot completion benchmark.
|
||||
This is different from mgsm_en which uses chat completions.
|
||||
|
||||
Model group is selected via AMD_TEST_MODEL_GROUP env var:
|
||||
- "gpt-oss": GPT-OSS models only (default, nightly-amd-8-gpu)
|
||||
- "grok": All GROK models (nightly-amd-8-gpu-grok)
|
||||
- "all": All models
|
||||
"""
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
# Get model group from environment
|
||||
cls.model_group = get_model_group()
|
||||
cls.models = get_models_for_group(cls.model_group)
|
||||
cls.base_url = DEFAULT_URL_FOR_TEST
|
||||
cls.num_questions = int(os.environ.get("GSM8K_NUM_QUESTIONS", "200"))
|
||||
|
||||
print(f"\n{'='*60}")
|
||||
print(f"AMD GSM8K Completion Evaluation Test")
|
||||
print(f"{'='*60}")
|
||||
print(f"Model group: {cls.model_group}")
|
||||
print(f"Models to test: {len(cls.models)}")
|
||||
for m in cls.models:
|
||||
print(f" - {m.model_path}")
|
||||
print(f"Questions per model: {cls.num_questions}")
|
||||
print(f"{'='*60}\n")
|
||||
|
||||
def test_gsm8k_completion_all_models(self):
|
||||
"""Test all configured base models with GSM8K completion benchmark."""
|
||||
all_results = []
|
||||
total_test_start = time.time()
|
||||
|
||||
# Summary table with runtime columns
|
||||
summary = f"### Model Group: {self.model_group}\n\n"
|
||||
summary += (
|
||||
"| Model | TP | Accuracy | Threshold | Startup | Bench | Total | Status |\n"
|
||||
)
|
||||
summary += (
|
||||
"| ----- | -- | -------- | --------- | ------- | ----- | ----- | ------ |\n"
|
||||
)
|
||||
|
||||
for config in self.models:
|
||||
display_name = config.get_display_name()
|
||||
with self.subTest(model=display_name):
|
||||
print(f"\n{'='*60}")
|
||||
print(f"Testing: {display_name} (TP={config.tp_size})")
|
||||
print(f"{'='*60}")
|
||||
|
||||
error_message = None
|
||||
acc, invalid, latency = None, None, None
|
||||
startup_time, bench_time, total_time = None, None, None
|
||||
skipped = False
|
||||
model_start = time.time()
|
||||
|
||||
# Check model availability with detailed logging
|
||||
is_available, status_msg = log_model_status(config)
|
||||
|
||||
if not is_available:
|
||||
print(f"\n❌ MODEL NOT AVAILABLE: {status_msg}")
|
||||
print(f"⏭️ SKIPPING: {display_name}")
|
||||
status = f"⏭️ SKIP"
|
||||
skipped = True
|
||||
all_results.append(
|
||||
{
|
||||
"model": display_name,
|
||||
"tp_size": config.tp_size,
|
||||
"accuracy": None,
|
||||
"threshold": config.accuracy_threshold,
|
||||
"invalid": None,
|
||||
"latency": None,
|
||||
"startup_time": None,
|
||||
"bench_time": None,
|
||||
"total_time": None,
|
||||
"passed": True, # Don't count as failure
|
||||
"skipped": True,
|
||||
"error": status_msg,
|
||||
}
|
||||
)
|
||||
else:
|
||||
try:
|
||||
# Launch server with timing
|
||||
print(f"\n🚀 Launching server for {display_name}...")
|
||||
server_start = time.time()
|
||||
process = popen_launch_server_for_base_model(
|
||||
self.base_url, config
|
||||
)
|
||||
startup_time = time.time() - server_start
|
||||
print(f"⏱️ Server startup: {startup_time:.1f}s")
|
||||
|
||||
try:
|
||||
# Run benchmark with timing and retries
|
||||
print(
|
||||
f"📊 Running GSM8K benchmark ({self.num_questions} questions)..."
|
||||
)
|
||||
bench_start = time.time()
|
||||
acc, invalid, latency = None, None, None
|
||||
for attempt in range(3):
|
||||
try:
|
||||
acc, invalid, latency = run_gsm8k_benchmark(
|
||||
self.base_url,
|
||||
num_questions=self.num_questions,
|
||||
num_shots=5,
|
||||
parallel=64,
|
||||
)
|
||||
print(
|
||||
f" Attempt {attempt + 1}: accuracy={acc:.3f}"
|
||||
)
|
||||
if acc >= config.accuracy_threshold:
|
||||
break
|
||||
except Exception as e:
|
||||
print(
|
||||
f" Attempt {attempt + 1} failed with error: {e}"
|
||||
)
|
||||
if attempt == 2:
|
||||
raise
|
||||
bench_time = time.time() - bench_start
|
||||
|
||||
total_time = time.time() - model_start
|
||||
|
||||
print(f"\n📈 Results for {display_name}:")
|
||||
print(
|
||||
f" Accuracy: {acc:.3f} (threshold: {config.accuracy_threshold})"
|
||||
)
|
||||
print(f" Invalid: {invalid:.3f}")
|
||||
print(f" Benchmark latency: {latency:.1f}s")
|
||||
print(f"\n⏱️ Runtime breakdown:")
|
||||
print(f" Server startup: {startup_time:.1f}s")
|
||||
print(f" Benchmark: {bench_time:.1f}s")
|
||||
print(f" Total: {total_time:.1f}s")
|
||||
|
||||
passed = acc >= config.accuracy_threshold
|
||||
status = "✅ PASS" if passed else "❌ FAIL"
|
||||
|
||||
if passed:
|
||||
print(f"\n Status: ✅ PASSED")
|
||||
else:
|
||||
print(f"\n Status: ❌ FAILED (below threshold)")
|
||||
|
||||
all_results.append(
|
||||
{
|
||||
"model": display_name,
|
||||
"tp_size": config.tp_size,
|
||||
"accuracy": acc,
|
||||
"threshold": config.accuracy_threshold,
|
||||
"invalid": invalid,
|
||||
"latency": latency,
|
||||
"startup_time": startup_time,
|
||||
"bench_time": bench_time,
|
||||
"total_time": total_time,
|
||||
"passed": passed,
|
||||
"skipped": False,
|
||||
"error": None,
|
||||
}
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
error_message = str(e)
|
||||
total_time = time.time() - model_start
|
||||
print(f"\n❌ Error during benchmark: {error_message}")
|
||||
status = "❌ ERROR"
|
||||
all_results.append(
|
||||
{
|
||||
"model": display_name,
|
||||
"tp_size": config.tp_size,
|
||||
"accuracy": None,
|
||||
"threshold": config.accuracy_threshold,
|
||||
"invalid": None,
|
||||
"latency": None,
|
||||
"startup_time": startup_time,
|
||||
"bench_time": None,
|
||||
"total_time": total_time,
|
||||
"passed": False,
|
||||
"skipped": False,
|
||||
"error": error_message,
|
||||
}
|
||||
)
|
||||
|
||||
finally:
|
||||
print(f"\n🛑 Stopping server for {display_name}...")
|
||||
kill_process_tree(process.pid)
|
||||
|
||||
except Exception as e:
|
||||
error_message = str(e)
|
||||
total_time = time.time() - model_start
|
||||
print(f"\n❌ Error launching server: {error_message}")
|
||||
status = "❌ ERROR"
|
||||
all_results.append(
|
||||
{
|
||||
"model": display_name,
|
||||
"tp_size": config.tp_size,
|
||||
"accuracy": None,
|
||||
"threshold": config.accuracy_threshold,
|
||||
"invalid": None,
|
||||
"latency": None,
|
||||
"startup_time": None,
|
||||
"bench_time": None,
|
||||
"total_time": total_time,
|
||||
"passed": False,
|
||||
"skipped": False,
|
||||
"error": error_message,
|
||||
}
|
||||
)
|
||||
|
||||
# Add to summary with runtime (use display name to show variant)
|
||||
acc_str = f"{acc:.3f}" if acc is not None else "N/A"
|
||||
startup_str = (
|
||||
f"{startup_time:.0f}s" if startup_time is not None else "N/A"
|
||||
)
|
||||
bench_str = f"{bench_time:.0f}s" if bench_time is not None else "N/A"
|
||||
total_str = f"{total_time:.0f}s" if total_time is not None else "N/A"
|
||||
summary += f"| {display_name} | {config.tp_size} | {acc_str} | {config.accuracy_threshold} | {startup_str} | {bench_str} | {total_str} | {status} |\n"
|
||||
|
||||
# Calculate total test runtime
|
||||
total_test_time = time.time() - total_test_start
|
||||
|
||||
# Print summary
|
||||
print(f"\n{'='*60}")
|
||||
print(f"SUMMARY - Model Group: {self.model_group}")
|
||||
print(f"{'='*60}")
|
||||
print(summary)
|
||||
print(
|
||||
f"\n⏱️ Total test runtime: {total_test_time:.1f}s ({total_test_time/60:.1f} min)"
|
||||
)
|
||||
|
||||
# Check for failures (exclude skipped models)
|
||||
failed_models = [
|
||||
r for r in all_results if not r["passed"] and not r.get("skipped", False)
|
||||
]
|
||||
skipped_models = [r for r in all_results if r.get("skipped", False)]
|
||||
passed_models = [
|
||||
r for r in all_results if r["passed"] and not r.get("skipped", False)
|
||||
]
|
||||
|
||||
# Build GitHub summary with results and failure details
|
||||
# Note: summary already includes the "### Model Group:" header
|
||||
github_summary = f"{summary}\n"
|
||||
github_summary += f"\n**Statistics:** ✅ Passed: {len(passed_models)} | ❌ Failed: {len(failed_models)} | ⏭️ Skipped: {len(skipped_models)}\n"
|
||||
github_summary += f"\n**Total Runtime:** {total_test_time:.1f}s ({total_test_time/60:.1f} min)\n"
|
||||
|
||||
if failed_models:
|
||||
github_summary += "\n#### ❌ Failed Models\n"
|
||||
for r in failed_models:
|
||||
acc_str = f"{r['accuracy']:.3f}" if r["accuracy"] is not None else "N/A"
|
||||
github_summary += f"- **{r['model']}**: accuracy={acc_str}, threshold={r['threshold']}"
|
||||
if r.get("error"):
|
||||
# Truncate long errors for display
|
||||
error_short = (
|
||||
r["error"][:200] + "..."
|
||||
if len(r["error"]) > 200
|
||||
else r["error"]
|
||||
)
|
||||
github_summary += f"\n - Error: `{error_short}`"
|
||||
github_summary += "\n"
|
||||
|
||||
if skipped_models:
|
||||
github_summary += "\n#### ⏭️ Skipped Models\n"
|
||||
for r in skipped_models:
|
||||
github_summary += (
|
||||
f"- **{r['model']}**: {r.get('error', 'Not available')}\n"
|
||||
)
|
||||
|
||||
# Write GitHub step summary
|
||||
if is_in_ci():
|
||||
write_github_step_summary(github_summary)
|
||||
|
||||
print(f"\n📊 Final Statistics:")
|
||||
print(f" Passed: {len(passed_models)}")
|
||||
print(f" Failed: {len(failed_models)}")
|
||||
print(f" Skipped: {len(skipped_models)}")
|
||||
|
||||
if skipped_models:
|
||||
print(f"\n⏭️ Skipped models (not available):")
|
||||
for r in skipped_models:
|
||||
print(f" - {r['model']}: {r['error']}")
|
||||
|
||||
if failed_models:
|
||||
print(f"\n❌ Failed models:")
|
||||
for r in failed_models:
|
||||
acc_str = f"{r['accuracy']:.3f}" if r["accuracy"] is not None else "N/A"
|
||||
print(
|
||||
f" - {r['model']}: accuracy={acc_str}, threshold={r['threshold']}"
|
||||
)
|
||||
if r.get("error"):
|
||||
print(f" Error: {r['error'][:200]}")
|
||||
|
||||
failure_msg = "\n".join(
|
||||
[
|
||||
f"- {r['model']}: accuracy={r['accuracy']}, threshold={r['threshold']}, error={r['error']}"
|
||||
for r in failed_models
|
||||
]
|
||||
)
|
||||
raise AssertionError(f"The following models failed:\n{failure_msg}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -1,726 +0,0 @@
|
||||
"""
|
||||
MI35x GSM8K Completion Evaluation Test (8-GPU)
|
||||
|
||||
This test uses the completion-based gsm8k benchmark (few-shot prompting)
|
||||
for MI35x-specific models that differ from MI300X configurations.
|
||||
|
||||
MI35x-specific models:
|
||||
- GPT-OSS series: Uses openai/gpt-oss-* (not lmsys/gpt-oss-*-bf16)
|
||||
- DeepSeek-R1-0528: Same model as MI300X (MXFP4 only used for perf tests)
|
||||
|
||||
Model groups are selected via AMD_TEST_MODEL_GROUP environment variable:
|
||||
- "gpt-oss" (default): GPT-OSS models with MI35x paths
|
||||
- "deepseek-r1": DeepSeek-R1-0528 basic + MTP (same as MI300X)
|
||||
- "deepseek-r1-dp-tc": DeepSeek-R1-0528 DP + TC (same as MI300X)
|
||||
- "deepseek-r1-all": All DeepSeek-R1-0528 variants (basic, MTP, DP, TC)
|
||||
|
||||
Registry: nightly-amd-8-gpu-mi35x suite (8-GPU tests on MI35x)
|
||||
"""
|
||||
|
||||
import ast
|
||||
import os
|
||||
|
||||
# Set HF cache to /data2/models/ for MI35x so HF models download there
|
||||
os.environ.setdefault("HF_HOME", "/data2/models/huggingface")
|
||||
os.environ.setdefault("HF_HUB_CACHE", "/data2/models/huggingface/hub")
|
||||
import re
|
||||
import subprocess
|
||||
import time
|
||||
import unittest
|
||||
from dataclasses import dataclass
|
||||
from typing import List, Optional, Tuple
|
||||
|
||||
import numpy as np
|
||||
|
||||
# HuggingFace Hub for model cache checking and download progress
|
||||
try:
|
||||
from huggingface_hub import HfFileSystem
|
||||
from huggingface_hub.utils import GatedRepoError, RepositoryNotFoundError
|
||||
|
||||
HF_HUB_AVAILABLE = True
|
||||
except ImportError:
|
||||
HF_HUB_AVAILABLE = False
|
||||
print("[WARNING] huggingface_hub not available - model cache checking disabled")
|
||||
|
||||
from sglang.srt.utils import kill_process_tree
|
||||
from sglang.test.ci.ci_register import register_amd_ci
|
||||
from sglang.test.test_utils import (
|
||||
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||
DEFAULT_URL_FOR_TEST,
|
||||
is_in_ci,
|
||||
popen_launch_server,
|
||||
write_github_step_summary,
|
||||
)
|
||||
from sglang.utils import download_and_cache_file, read_jsonl
|
||||
|
||||
# Register for AMD CI - MI35x 8-GPU GSM8K completion tests (~120 min)
|
||||
register_amd_ci(est_time=7200, suite="nightly-amd-8-gpu-mi35x", nightly=True)
|
||||
|
||||
INVALID = -9999999
|
||||
|
||||
|
||||
@dataclass
|
||||
class BaseModelConfig:
|
||||
"""Configuration for a base model to test."""
|
||||
|
||||
model_path: str # HuggingFace model ID (e.g., "amd/DeepSeek-R1-MXFP4-Preview")
|
||||
tp_size: int = 8
|
||||
accuracy_threshold: float = 0.50
|
||||
other_args: Optional[List[str]] = None
|
||||
env_vars: Optional[dict] = None
|
||||
tokenizer_path: Optional[str] = None
|
||||
timeout: Optional[int] = None
|
||||
local_path: Optional[str] = None # Preferred local path (checked first before HF)
|
||||
variant: Optional[str] = (
|
||||
None # Test variant name (e.g., "basic", "MTP", "DP", "TC")
|
||||
)
|
||||
|
||||
def __post_init__(self):
|
||||
if self.other_args is None:
|
||||
self.other_args = []
|
||||
if self.env_vars is None:
|
||||
self.env_vars = {}
|
||||
|
||||
def get_effective_model_path(self) -> str:
|
||||
"""Return local_path if it exists, otherwise model_path (HF ID)."""
|
||||
if self.local_path and os.path.exists(self.local_path):
|
||||
return self.local_path
|
||||
return self.model_path
|
||||
|
||||
def get_display_name(self) -> str:
|
||||
"""Return display name for logs/summary (model + variant if set)."""
|
||||
if self.variant:
|
||||
return f"{self.model_path} ({self.variant})"
|
||||
return self.model_path
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# MI35x MODEL GROUPS - Different from MI300X configurations
|
||||
# =============================================================================
|
||||
|
||||
# Group 1: GPT-OSS models (MI35x uses openai/* paths, not lmsys/*)
|
||||
MI35X_GPT_OSS_MODELS = [
|
||||
# GPT-OSS-20B - MI35x specific path
|
||||
BaseModelConfig(
|
||||
model_path="openai/gpt-oss-20b",
|
||||
tp_size=8,
|
||||
accuracy_threshold=0.47,
|
||||
other_args=[
|
||||
"--chunked-prefill-size",
|
||||
"130172",
|
||||
"--max-running-requests",
|
||||
"128",
|
||||
"--mem-fraction-static",
|
||||
"0.85",
|
||||
"--attention-backend",
|
||||
"triton",
|
||||
"--trust-remote-code",
|
||||
],
|
||||
env_vars={"SGLANG_USE_AITER": "1"},
|
||||
),
|
||||
# GPT-OSS-120B - MI35x specific path
|
||||
BaseModelConfig(
|
||||
model_path="openai/gpt-oss-120b",
|
||||
tp_size=8,
|
||||
accuracy_threshold=0.79,
|
||||
timeout=900, # 15 minutes for 120B model
|
||||
other_args=[
|
||||
"--chunked-prefill-size",
|
||||
"130172",
|
||||
"--max-running-requests",
|
||||
"128",
|
||||
"--mem-fraction-static",
|
||||
"0.85",
|
||||
"--attention-backend",
|
||||
"triton",
|
||||
"--trust-remote-code",
|
||||
],
|
||||
env_vars={"SGLANG_USE_AITER": "1"},
|
||||
),
|
||||
]
|
||||
|
||||
# Group 2: DeepSeek-R1-0528 basic + MTP (same model as MI300X for consistency)
|
||||
# Runner: nightly-test-8-gpu-mi35x-deepseek-r1
|
||||
# Note: MXFP4 variant only used for perf tests (test_deepseek_r1_mxfp4_perf.py)
|
||||
MI35X_DEEPSEEK_R1_MODELS = [
|
||||
# DeepSeek-R1-0528 basic - reasoning model, ~80GB per GPU
|
||||
BaseModelConfig(
|
||||
model_path="deepseek-ai/DeepSeek-R1-0528",
|
||||
tp_size=8,
|
||||
accuracy_threshold=0.93,
|
||||
timeout=3600, # 1 hour for large model
|
||||
variant="basic",
|
||||
other_args=[
|
||||
"--attention-backend",
|
||||
"aiter",
|
||||
"--chunked-prefill-size",
|
||||
"131072",
|
||||
"--disable-radix-cache",
|
||||
"--mem-fraction-static",
|
||||
"0.85",
|
||||
"--trust-remote-code",
|
||||
],
|
||||
env_vars={
|
||||
"SGLANG_USE_AITER": "1",
|
||||
},
|
||||
),
|
||||
# DeepSeek-R1-0528 with MTP (EAGLE speculative decoding)
|
||||
BaseModelConfig(
|
||||
model_path="deepseek-ai/DeepSeek-R1-0528",
|
||||
tp_size=8,
|
||||
accuracy_threshold=0.93,
|
||||
timeout=3600,
|
||||
variant="MTP",
|
||||
other_args=[
|
||||
"--chunked-prefill-size",
|
||||
"131072",
|
||||
"--speculative-algorithm",
|
||||
"EAGLE",
|
||||
"--speculative-num-steps",
|
||||
"3",
|
||||
"--speculative-eagle-topk",
|
||||
"1",
|
||||
"--speculative-num-draft-tokens",
|
||||
"4",
|
||||
"--mem-fraction-static",
|
||||
"0.7",
|
||||
"--trust-remote-code",
|
||||
],
|
||||
env_vars={
|
||||
"SGLANG_USE_AITER": "1",
|
||||
},
|
||||
),
|
||||
]
|
||||
|
||||
# Group 3: DeepSeek-R1-0528 with DP + TC (requires ROCm 7.0+)
|
||||
# Runner: nightly-test-8-gpu-mi35x-deepseek-r1-dp-tc
|
||||
MI35X_DEEPSEEK_R1_DP_TC_MODELS = [
|
||||
# DeepSeek-R1-0528 with DP attention
|
||||
BaseModelConfig(
|
||||
model_path="deepseek-ai/DeepSeek-R1-0528",
|
||||
tp_size=8,
|
||||
accuracy_threshold=0.93,
|
||||
timeout=3600,
|
||||
variant="DP",
|
||||
other_args=[
|
||||
"--chunked-prefill-size",
|
||||
"131072",
|
||||
"--dp-size",
|
||||
"8",
|
||||
"--enable-dp-attention",
|
||||
"--mem-fraction-static",
|
||||
"0.85",
|
||||
"--trust-remote-code",
|
||||
],
|
||||
env_vars={
|
||||
"SGLANG_USE_ROCM700A": "1",
|
||||
"SGLANG_USE_AITER": "1",
|
||||
},
|
||||
),
|
||||
# DeepSeek-R1-0528 with torch compile
|
||||
BaseModelConfig(
|
||||
model_path="deepseek-ai/DeepSeek-R1-0528",
|
||||
tp_size=8,
|
||||
accuracy_threshold=0.93,
|
||||
timeout=7200, # 2 hours for compilation
|
||||
variant="TC",
|
||||
other_args=[
|
||||
"--chunked-prefill-size",
|
||||
"131072",
|
||||
"--mem-fraction-static",
|
||||
"0.70",
|
||||
"--cuda-graph-max-bs",
|
||||
"8",
|
||||
"--enable-torch-compile",
|
||||
"--disable-cuda-graph",
|
||||
"--trust-remote-code",
|
||||
],
|
||||
env_vars={
|
||||
"SGLANG_USE_ROCM700A": "1",
|
||||
"SGLANG_USE_AITER": "1",
|
||||
},
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def get_model_group() -> str:
|
||||
"""Get the model group to test from environment variable."""
|
||||
return os.environ.get("AMD_TEST_MODEL_GROUP", "gpt-oss")
|
||||
|
||||
|
||||
def get_models_for_group(group: str) -> List[BaseModelConfig]:
|
||||
"""Get the list of models for a given group.
|
||||
|
||||
Note: DeepSeek-R1-MXFP4 is only used for perf tests, not accuracy tests.
|
||||
See test_deepseek_r1_mxfp4_perf.py for MXFP4 perf tests.
|
||||
"""
|
||||
if group == "gpt-oss":
|
||||
return MI35X_GPT_OSS_MODELS
|
||||
elif group == "deepseek-r1":
|
||||
return MI35X_DEEPSEEK_R1_MODELS
|
||||
elif group == "deepseek-r1-dp-tc":
|
||||
return MI35X_DEEPSEEK_R1_DP_TC_MODELS
|
||||
elif group == "deepseek-r1-all":
|
||||
# All DeepSeek-R1-0528 variants: basic, MTP, DP, TC
|
||||
return MI35X_DEEPSEEK_R1_MODELS + MI35X_DEEPSEEK_R1_DP_TC_MODELS
|
||||
elif group == "all":
|
||||
return (
|
||||
MI35X_GPT_OSS_MODELS
|
||||
+ MI35X_DEEPSEEK_R1_MODELS
|
||||
+ MI35X_DEEPSEEK_R1_DP_TC_MODELS
|
||||
)
|
||||
else:
|
||||
print(f"[WARNING] Unknown model group '{group}', using 'gpt-oss'")
|
||||
return MI35X_GPT_OSS_MODELS
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# MODEL CACHE AND DOWNLOAD UTILITIES
|
||||
# =============================================================================
|
||||
|
||||
|
||||
def check_local_cache(model_path: str) -> Tuple[bool, str]:
|
||||
"""
|
||||
Check if model is cached locally.
|
||||
|
||||
Returns:
|
||||
Tuple of (is_cached, cache_path_or_message)
|
||||
"""
|
||||
# Check common HF cache locations for MI35x
|
||||
cache_dirs = [
|
||||
os.path.expanduser("~/.cache/huggingface/hub"),
|
||||
"/data2/models/huggingface/hub",
|
||||
os.environ.get("HF_HUB_CACHE", ""),
|
||||
]
|
||||
cache_dirs = [d for d in cache_dirs if d] # Remove empty
|
||||
|
||||
# Convert model_path to cache directory format (org--model)
|
||||
cache_name = f"models--{model_path.replace('/', '--')}"
|
||||
|
||||
for cache_dir in cache_dirs:
|
||||
cache_path = os.path.join(cache_dir, cache_name)
|
||||
if os.path.exists(cache_path):
|
||||
# Check if there are snapshots
|
||||
snapshots_dir = os.path.join(cache_path, "snapshots")
|
||||
if os.path.exists(snapshots_dir) and os.listdir(snapshots_dir):
|
||||
return True, cache_path
|
||||
|
||||
return False, f"Not found in: {', '.join(cache_dirs)}"
|
||||
|
||||
|
||||
def check_hf_repo_access(model_path: str) -> Tuple[bool, str]:
|
||||
"""
|
||||
Check if HuggingFace repository is accessible.
|
||||
|
||||
Returns:
|
||||
Tuple of (is_accessible, message)
|
||||
"""
|
||||
if not HF_HUB_AVAILABLE:
|
||||
return True, "huggingface_hub not available, skipping access check"
|
||||
|
||||
try:
|
||||
fs = HfFileSystem()
|
||||
# Try to list files in the repo
|
||||
files = fs.ls(model_path, detail=False)
|
||||
if files:
|
||||
return True, f"Repository accessible ({len(files)} files)"
|
||||
else:
|
||||
return False, "Repository exists but is empty"
|
||||
except GatedRepoError:
|
||||
return False, "GATED REPO - requires authentication/approval"
|
||||
except RepositoryNotFoundError:
|
||||
return False, "REPO NOT FOUND on HuggingFace"
|
||||
except Exception as e:
|
||||
error_msg = str(e)
|
||||
if "401" in error_msg or "unauthorized" in error_msg.lower():
|
||||
return False, f"AUTH ERROR - may need HF_TOKEN: {error_msg[:100]}"
|
||||
elif "404" in error_msg:
|
||||
return False, f"NOT FOUND: {error_msg[:100]}"
|
||||
elif "timeout" in error_msg.lower() or "connection" in error_msg.lower():
|
||||
return False, f"NETWORK ERROR: {error_msg[:100]}"
|
||||
else:
|
||||
return False, f"ERROR: {error_msg[:100]}"
|
||||
|
||||
|
||||
def log_model_status(config: "BaseModelConfig") -> Tuple[bool, str]:
|
||||
"""
|
||||
Log detailed model availability status.
|
||||
|
||||
Checks in order:
|
||||
1. local_path (if specified) - preferred local path
|
||||
2. model_path as local path (if starts with /)
|
||||
3. model_path as HF model ID - check cache then HF access
|
||||
|
||||
Returns:
|
||||
Tuple of (is_available, status_message)
|
||||
"""
|
||||
model_path = config.model_path
|
||||
local_path = config.local_path
|
||||
|
||||
print(f"\n📦 Checking model: {model_path}")
|
||||
if local_path:
|
||||
print(f" (preferred local: {local_path})")
|
||||
print("-" * 50)
|
||||
|
||||
# Step 1: Check preferred local_path first (if specified)
|
||||
if local_path:
|
||||
if os.path.exists(local_path):
|
||||
print(f" ✅ LOCAL PATH: Found at {local_path}")
|
||||
return True, f"Local path exists at {local_path}"
|
||||
else:
|
||||
print(f" ⚠️ LOCAL PATH: Not found at {local_path}, trying HF fallback...")
|
||||
|
||||
# Step 2: For absolute paths (starting with /), check if exists
|
||||
if model_path.startswith("/"):
|
||||
if os.path.exists(model_path):
|
||||
print(f" ✅ LOCAL PATH: Found at {model_path}")
|
||||
return True, f"Local path exists at {model_path}"
|
||||
else:
|
||||
print(f" ❌ LOCAL PATH: Not found at {model_path}")
|
||||
return False, f"Local path not found at {model_path}"
|
||||
|
||||
# Step 3: For HF model IDs, check local cache first
|
||||
is_cached, cache_msg = check_local_cache(model_path)
|
||||
if is_cached:
|
||||
print(f" ✅ LOCAL CACHE: Found at {cache_msg}")
|
||||
return True, f"Cached locally at {cache_msg}"
|
||||
else:
|
||||
print(f" ⚠️ LOCAL CACHE: {cache_msg}")
|
||||
|
||||
# Step 4: Check HF repo access (will download if accessible)
|
||||
is_accessible, access_msg = check_hf_repo_access(model_path)
|
||||
if is_accessible:
|
||||
print(f" ✅ HF ACCESS: {access_msg}")
|
||||
print(
|
||||
f" 📥 Model will be downloaded from HuggingFace to {os.environ.get('HF_HOME', '~/.cache/huggingface')}"
|
||||
)
|
||||
return True, f"Will download from HF: {access_msg}"
|
||||
else:
|
||||
print(f" ❌ HF ACCESS: {access_msg}")
|
||||
return False, access_msg
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# BENCHMARK UTILITIES
|
||||
# =============================================================================
|
||||
|
||||
|
||||
def get_one_example(lines, i, include_answer):
|
||||
"""Format a single GSM8K example."""
|
||||
ret = "Question: " + lines[i]["question"] + "\nAnswer:"
|
||||
if include_answer:
|
||||
ret += " " + lines[i]["answer"]
|
||||
return ret
|
||||
|
||||
|
||||
def get_few_shot_examples(lines, k):
|
||||
"""Get k few-shot examples for prompting."""
|
||||
ret = ""
|
||||
for i in range(k):
|
||||
ret += get_one_example(lines, i, True) + "\n\n"
|
||||
return ret
|
||||
|
||||
|
||||
def get_answer_value(answer_str):
|
||||
"""Extract numerical answer from response."""
|
||||
answer_str = answer_str.replace(",", "")
|
||||
numbers = re.findall(r"\d+", answer_str)
|
||||
if len(numbers) < 1:
|
||||
return INVALID
|
||||
try:
|
||||
return ast.literal_eval(numbers[-1])
|
||||
except SyntaxError:
|
||||
return INVALID
|
||||
|
||||
|
||||
def run_gsm8k_benchmark(
|
||||
base_url: str,
|
||||
num_questions: int = 200,
|
||||
num_shots: int = 5,
|
||||
parallel: int = 64,
|
||||
) -> Tuple[float, float, float]:
|
||||
"""Run GSM8K few-shot completion benchmark."""
|
||||
import sglang as sgl
|
||||
from sglang.lang.backend.runtime_endpoint import RuntimeEndpoint
|
||||
|
||||
url = "https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/test.jsonl"
|
||||
data_path = download_and_cache_file(url)
|
||||
lines = list(read_jsonl(data_path))
|
||||
|
||||
few_shot_examples = get_few_shot_examples(lines, num_shots)
|
||||
|
||||
questions = []
|
||||
labels = []
|
||||
for i in range(len(lines[:num_questions])):
|
||||
questions.append(get_one_example(lines, i, False))
|
||||
labels.append(get_answer_value(lines[i]["answer"]))
|
||||
assert all(l != INVALID for l in labels)
|
||||
arguments = [{"question": q} for q in questions]
|
||||
|
||||
@sgl.function
|
||||
def few_shot_gsm8k(s, question):
|
||||
s += few_shot_examples + question
|
||||
s += sgl.gen(
|
||||
"answer", max_tokens=512, stop=["Question", "Assistant:", "<|separator|>"]
|
||||
)
|
||||
|
||||
backend = RuntimeEndpoint(base_url)
|
||||
sgl.set_default_backend(backend)
|
||||
|
||||
tic = time.perf_counter()
|
||||
states = few_shot_gsm8k.run_batch(
|
||||
arguments,
|
||||
temperature=0,
|
||||
num_threads=parallel,
|
||||
progress_bar=True,
|
||||
)
|
||||
latency = time.perf_counter() - tic
|
||||
|
||||
preds = []
|
||||
for i in range(len(states)):
|
||||
preds.append(get_answer_value(states[i]["answer"]))
|
||||
|
||||
acc = np.mean(np.array(preds) == np.array(labels))
|
||||
invalid = np.mean(np.array(preds) == INVALID)
|
||||
|
||||
return float(acc), float(invalid), float(latency)
|
||||
|
||||
|
||||
def popen_launch_server_for_base_model(
|
||||
base_url: str,
|
||||
config: BaseModelConfig,
|
||||
) -> "subprocess.Popen":
|
||||
"""Launch server for a base model with appropriate configuration."""
|
||||
env = os.environ.copy()
|
||||
for key, value in config.env_vars.items():
|
||||
env[key] = value
|
||||
print(f"Setting env: {key}={value}")
|
||||
|
||||
other_args = list(config.other_args)
|
||||
other_args.extend(["--tp", str(config.tp_size)])
|
||||
other_args.extend(["--log-level-http", "warning"])
|
||||
|
||||
if config.tokenizer_path:
|
||||
other_args.extend(["--tokenizer-path", config.tokenizer_path])
|
||||
|
||||
timeout = config.timeout if config.timeout else DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH
|
||||
|
||||
# Use effective model path (local if exists, else HF model ID)
|
||||
effective_model_path = config.get_effective_model_path()
|
||||
print(f"Using model path: {effective_model_path}")
|
||||
|
||||
process = popen_launch_server(
|
||||
model=effective_model_path,
|
||||
base_url=base_url,
|
||||
timeout=timeout,
|
||||
other_args=other_args,
|
||||
env=env,
|
||||
)
|
||||
return process
|
||||
|
||||
|
||||
class TestMI35xGsm8kCompletionEval(unittest.TestCase):
|
||||
"""MI35x GSM8K Completion Evaluation Test (8-GPU)
|
||||
|
||||
Tests MI35x-specific base models using few-shot completion benchmark.
|
||||
"""
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.model_group = get_model_group()
|
||||
cls.models = get_models_for_group(cls.model_group)
|
||||
cls.base_url = DEFAULT_URL_FOR_TEST
|
||||
cls.num_questions = int(os.environ.get("GSM8K_NUM_QUESTIONS", "200"))
|
||||
|
||||
print(f"\n{'='*60}")
|
||||
print(f"MI35x GSM8K Completion Evaluation Test (8-GPU)")
|
||||
print(f"{'='*60}")
|
||||
print(f"Model group: {cls.model_group}")
|
||||
print(f"Models to test: {len(cls.models)}")
|
||||
for m in cls.models:
|
||||
print(f" - {m.model_path}")
|
||||
print(f"Questions per model: {cls.num_questions}")
|
||||
print(f"{'='*60}\n")
|
||||
|
||||
def test_gsm8k_completion_all_models(self):
|
||||
"""Test all configured MI35x models with GSM8K completion benchmark."""
|
||||
all_results = []
|
||||
total_test_start = time.time()
|
||||
|
||||
summary = f"### MI35x Model Group: {self.model_group}\n\n"
|
||||
summary += (
|
||||
"| Model | TP | Accuracy | Threshold | Startup | Bench | Total | Status |\n"
|
||||
)
|
||||
summary += (
|
||||
"| ----- | -- | -------- | --------- | ------- | ----- | ----- | ------ |\n"
|
||||
)
|
||||
|
||||
for config in self.models:
|
||||
display_name = config.get_display_name()
|
||||
with self.subTest(model=display_name):
|
||||
print(f"\n{'='*60}")
|
||||
print(f"Testing: {display_name} (TP={config.tp_size})")
|
||||
print(f"{'='*60}")
|
||||
|
||||
error_message = None
|
||||
acc, invalid, latency = None, None, None
|
||||
startup_time, bench_time, total_time = None, None, None
|
||||
model_start = time.time()
|
||||
|
||||
# Check model availability with detailed logging
|
||||
is_available, status_msg = log_model_status(config)
|
||||
|
||||
if not is_available:
|
||||
print(f"\n❌ MODEL NOT AVAILABLE: {status_msg}")
|
||||
print(f"⏭️ SKIPPING: {display_name}")
|
||||
status = "⏭️ SKIP"
|
||||
all_results.append(
|
||||
{
|
||||
"model": display_name,
|
||||
"tp_size": config.tp_size,
|
||||
"accuracy": None,
|
||||
"threshold": config.accuracy_threshold,
|
||||
"passed": True,
|
||||
"skipped": True,
|
||||
"error": status_msg,
|
||||
}
|
||||
)
|
||||
else:
|
||||
try:
|
||||
print(f"\n🚀 Launching server for {display_name}...")
|
||||
server_start = time.time()
|
||||
process = popen_launch_server_for_base_model(
|
||||
self.base_url, config
|
||||
)
|
||||
startup_time = time.time() - server_start
|
||||
print(f"⏱️ Server startup: {startup_time:.1f}s")
|
||||
|
||||
try:
|
||||
print(
|
||||
f"📊 Running GSM8K benchmark ({self.num_questions} questions)..."
|
||||
)
|
||||
bench_start = time.time()
|
||||
for attempt in range(3):
|
||||
try:
|
||||
acc, invalid, latency = run_gsm8k_benchmark(
|
||||
self.base_url,
|
||||
num_questions=self.num_questions,
|
||||
num_shots=5,
|
||||
parallel=64,
|
||||
)
|
||||
print(
|
||||
f" Attempt {attempt + 1}: accuracy={acc:.3f}"
|
||||
)
|
||||
if acc >= config.accuracy_threshold:
|
||||
break
|
||||
except Exception as e:
|
||||
print(f" Attempt {attempt + 1} failed: {e}")
|
||||
if attempt == 2:
|
||||
raise
|
||||
bench_time = time.time() - bench_start
|
||||
total_time = time.time() - model_start
|
||||
|
||||
passed = acc >= config.accuracy_threshold
|
||||
status = "✅ PASS" if passed else "❌ FAIL"
|
||||
|
||||
print(
|
||||
f"\n📈 Results: accuracy={acc:.3f} (threshold: {config.accuracy_threshold})"
|
||||
)
|
||||
print(f"⏱️ Total: {total_time:.1f}s")
|
||||
|
||||
all_results.append(
|
||||
{
|
||||
"model": display_name,
|
||||
"tp_size": config.tp_size,
|
||||
"accuracy": acc,
|
||||
"threshold": config.accuracy_threshold,
|
||||
"startup_time": startup_time,
|
||||
"bench_time": bench_time,
|
||||
"total_time": total_time,
|
||||
"passed": passed,
|
||||
"skipped": False,
|
||||
"error": None,
|
||||
}
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
error_message = str(e)
|
||||
total_time = time.time() - model_start
|
||||
print(f"\n❌ Error: {error_message}")
|
||||
status = "❌ ERROR"
|
||||
all_results.append(
|
||||
{
|
||||
"model": display_name,
|
||||
"tp_size": config.tp_size,
|
||||
"accuracy": None,
|
||||
"threshold": config.accuracy_threshold,
|
||||
"passed": False,
|
||||
"skipped": False,
|
||||
"error": error_message,
|
||||
}
|
||||
)
|
||||
|
||||
finally:
|
||||
print(f"\n🛑 Stopping server...")
|
||||
kill_process_tree(process.pid)
|
||||
|
||||
except Exception as e:
|
||||
error_message = str(e)
|
||||
total_time = time.time() - model_start
|
||||
print(f"\n❌ Error launching server: {error_message}")
|
||||
status = "❌ ERROR"
|
||||
all_results.append(
|
||||
{
|
||||
"model": display_name,
|
||||
"tp_size": config.tp_size,
|
||||
"accuracy": None,
|
||||
"threshold": config.accuracy_threshold,
|
||||
"passed": False,
|
||||
"skipped": False,
|
||||
"error": error_message,
|
||||
}
|
||||
)
|
||||
|
||||
# Add to summary (use display name to show variant)
|
||||
acc_str = f"{acc:.3f}" if acc is not None else "N/A"
|
||||
startup_str = (
|
||||
f"{startup_time:.0f}s" if startup_time is not None else "N/A"
|
||||
)
|
||||
bench_str = f"{bench_time:.0f}s" if bench_time is not None else "N/A"
|
||||
total_str = f"{total_time:.0f}s" if total_time is not None else "N/A"
|
||||
summary += f"| {display_name} | {config.tp_size} | {acc_str} | {config.accuracy_threshold} | {startup_str} | {bench_str} | {total_str} | {status} |\n"
|
||||
|
||||
# Final summary
|
||||
total_test_time = time.time() - total_test_start
|
||||
failed_models = [
|
||||
r for r in all_results if not r["passed"] and not r.get("skipped", False)
|
||||
]
|
||||
skipped_models = [r for r in all_results if r.get("skipped", False)]
|
||||
passed_models = [
|
||||
r for r in all_results if r["passed"] and not r.get("skipped", False)
|
||||
]
|
||||
|
||||
print(f"\n{'='*60}")
|
||||
print(f"SUMMARY - MI35x Model Group: {self.model_group}")
|
||||
print(f"{'='*60}")
|
||||
print(summary)
|
||||
print(
|
||||
f"\n📊 Passed: {len(passed_models)} | Failed: {len(failed_models)} | Skipped: {len(skipped_models)}"
|
||||
)
|
||||
print(f"⏱️ Total: {total_test_time:.1f}s ({total_test_time/60:.1f} min)")
|
||||
|
||||
if is_in_ci():
|
||||
write_github_step_summary(summary)
|
||||
|
||||
if failed_models:
|
||||
failure_msg = "\n".join(
|
||||
[
|
||||
f"- {r['model']}: {r.get('error', 'below threshold')}"
|
||||
for r in failed_models
|
||||
]
|
||||
)
|
||||
raise AssertionError(f"The following models failed:\n{failure_msg}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -1,11 +1,13 @@
|
||||
"""Nightly performance benchmark for DeepSeek-R1-MXFP4 model (MI35x).
|
||||
"""MI35x Nightly performance benchmark for DeepSeek-R1-MXFP4 model.
|
||||
|
||||
This test benchmarks the DeepSeek-R1-MXFP4 quantized model on MI35x with 8 GPUs.
|
||||
|
||||
The model path can be configured via DEEPSEEK_R1_MXFP4_MODEL_PATH environment variable.
|
||||
|
||||
Registry: nightly-perf-8-gpu-mi35x-deepseek-r1-mxfp4 suite
|
||||
|
||||
Example usage:
|
||||
DEEPSEEK_R1_MXFP4_MODEL_PATH=/data2/models/amd-DeepSeek-R1-MXFP4-Preview python -m pytest test_deepseek_r1_mxfp4_perf.py -v
|
||||
DEEPSEEK_R1_MXFP4_MODEL_PATH=/data2/models/amd-DeepSeek-R1-MXFP4-Preview python -m pytest test_deepseek_r1_mxfp4_perf_mi35x.py -v
|
||||
"""
|
||||
|
||||
import os
|
||||
@@ -21,9 +23,9 @@ from sglang.test.nightly_bench_utils import BenchmarkResult
|
||||
from sglang.test.nightly_utils import NightlyBenchmarkRunner
|
||||
from sglang.test.test_utils import DEFAULT_URL_FOR_TEST, _parse_int_list_env
|
||||
|
||||
# Register for AMD CI - DeepSeek-R1-MXFP4 benchmark (~300 min)
|
||||
# Register for AMD CI - DeepSeek-R1-MXFP4 benchmark on MI35x (~300 min)
|
||||
register_amd_ci(
|
||||
est_time=18000, suite="nightly-perf-8-gpu-deepseek-r1-mxfp4", nightly=True
|
||||
est_time=18000, suite="nightly-perf-8-gpu-mi35x-deepseek-r1-mxfp4", nightly=True
|
||||
)
|
||||
|
||||
|
||||
@@ -33,7 +35,7 @@ def generate_simple_markdown_report(results: List[BenchmarkResult]) -> str:
|
||||
if results[0].run_name and results[0].run_name != "default":
|
||||
model_header += f" ({results[0].run_name})"
|
||||
|
||||
gpu_config = os.getenv("GPU_CONFIG", "")
|
||||
gpu_config = os.getenv("GPU_CONFIG", "MI35x")
|
||||
if gpu_config:
|
||||
model_header += f" [{gpu_config}]"
|
||||
|
||||
@@ -52,7 +54,7 @@ def generate_simple_markdown_report(results: List[BenchmarkResult]) -> str:
|
||||
# Priority: 1) env var, 2) local path, 3) HuggingFace model ID
|
||||
DEEPSEEK_R1_MXFP4_LOCAL_PATH = "/data2/models/amd-DeepSeek-R1-MXFP4-Preview"
|
||||
DEEPSEEK_R1_MXFP4_HF_MODEL_ID = "amd/DeepSeek-R1-MXFP4-Preview"
|
||||
PROFILE_DIR = "performance_profiles_deepseek_r1_mxfp4"
|
||||
PROFILE_DIR = "performance_profiles_deepseek_r1_mxfp4_mi35x"
|
||||
|
||||
|
||||
def get_model_path() -> str:
|
||||
@@ -68,8 +70,8 @@ def get_model_path() -> str:
|
||||
return DEEPSEEK_R1_MXFP4_HF_MODEL_ID
|
||||
|
||||
|
||||
class TestNightlyDeepseekR1MXFP4Performance(unittest.TestCase):
|
||||
"""Nightly performance benchmark for DeepSeek-R1-MXFP4 model (MI35x).
|
||||
class TestDeepseekR1MXFP4PerfMI35x(unittest.TestCase):
|
||||
"""MI35x Nightly performance benchmark for DeepSeek-R1-MXFP4 model.
|
||||
|
||||
Tests the DeepSeek-R1-MXFP4 quantized model on TP=8 with DP=8.
|
||||
Uses local path if available, otherwise downloads from HuggingFace.
|
||||
126
test/registered/amd/perf/mi35x/test_grok1_int4_perf_mi35x.py
Normal file
126
test/registered/amd/perf/mi35x/test_grok1_int4_perf_mi35x.py
Normal file
@@ -0,0 +1,126 @@
|
||||
"""MI35x Nightly performance benchmark for Grok-1 INT4 (W4A8KV8).
|
||||
|
||||
This test benchmarks Grok-1 (314B MOE) with INT4 weight quantization on 8 GPUs.
|
||||
|
||||
Registry: nightly-perf-8-gpu-mi35x-grok1-int4 suite
|
||||
"""
|
||||
|
||||
import os
|
||||
import unittest
|
||||
from typing import List
|
||||
|
||||
from sglang.test.ci.ci_register import register_amd_ci
|
||||
from sglang.test.nightly_bench_utils import BenchmarkResult
|
||||
from sglang.test.nightly_utils import NightlyBenchmarkRunner
|
||||
from sglang.test.test_utils import DEFAULT_URL_FOR_TEST, _parse_int_list_env
|
||||
|
||||
# Register for AMD CI - Grok-1 INT4 benchmark on MI35x (~25 min)
|
||||
register_amd_ci(
|
||||
est_time=1500, suite="nightly-perf-8-gpu-mi35x-grok1-int4", nightly=True
|
||||
)
|
||||
|
||||
|
||||
def generate_simple_markdown_report(results: List[BenchmarkResult]) -> str:
|
||||
"""Generate a simplified markdown report without traces and cost columns."""
|
||||
model_header = results[0].model_path
|
||||
if results[0].run_name and results[0].run_name != "default":
|
||||
model_header += f" ({results[0].run_name})"
|
||||
|
||||
gpu_config = os.getenv("GPU_CONFIG", "MI35x")
|
||||
if gpu_config:
|
||||
model_header += f" [{gpu_config}]"
|
||||
|
||||
summary = f"### {model_header}\n"
|
||||
summary += "| batch size | input len | latency (s) | input throughput (tok/s) | output throughput (tok/s) | ITL (ms) |\n"
|
||||
summary += "| ---------- | --------- | ----------- | ------------------------ | ------------------------- | -------- |\n"
|
||||
|
||||
for result in results:
|
||||
itl = 1 / (result.output_throughput / result.batch_size) * 1000
|
||||
summary += f"| {result.batch_size} | {result.input_len} | {result.latency:.2f} | {result.input_throughput:.2f} | {result.output_throughput:.2f} | {itl:.2f} |\n"
|
||||
|
||||
return summary
|
||||
|
||||
|
||||
# Model and tokenizer paths can be overridden via environment variables
|
||||
GROK1_MODEL_PATH = os.environ.get("GROK1_MODEL_PATH", "amd/grok-1-W4A8KV8")
|
||||
GROK1_TOKENIZER_PATH = os.environ.get("GROK1_TOKENIZER_PATH", "Xenova/grok-1-tokenizer")
|
||||
PROFILE_DIR = "performance_profiles_grok1_int4_mi35x"
|
||||
|
||||
|
||||
class TestGrok1INT4PerfMI35x(unittest.TestCase):
|
||||
"""Test suite for Grok-1 INT4 performance benchmarks on MI35x."""
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.base_url = DEFAULT_URL_FOR_TEST
|
||||
cls.batch_sizes = [1, 1, 8, 16, 64]
|
||||
cls.input_lens = tuple(_parse_int_list_env("NIGHTLY_INPUT_LENS", "1024"))
|
||||
cls.output_lens = tuple(_parse_int_list_env("NIGHTLY_OUTPUT_LENS", "512"))
|
||||
|
||||
cls.model_config = {
|
||||
"name": "grok1-int4-mi35x",
|
||||
"model_path": GROK1_MODEL_PATH,
|
||||
"other_args": [
|
||||
"--trust-remote-code",
|
||||
"--tp",
|
||||
"8",
|
||||
"--quantization",
|
||||
"fp8",
|
||||
"--mem-fraction-static",
|
||||
"0.85",
|
||||
"--tokenizer-path",
|
||||
GROK1_TOKENIZER_PATH,
|
||||
"--attention-backend",
|
||||
"aiter",
|
||||
],
|
||||
"env_vars": {
|
||||
"RCCL_MSCCL_ENABLE": "0",
|
||||
"SGLANG_USE_AITER": "1",
|
||||
"SGLANG_INT4_WEIGHT": "1",
|
||||
},
|
||||
}
|
||||
|
||||
cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url)
|
||||
cls.runner.setup_profile_directory()
|
||||
cls.runner.full_report = f"## {cls.__name__}\n"
|
||||
|
||||
def test_grok1_int4_perf(self):
|
||||
"""Run Grok-1 INT4 performance benchmark on MI35x."""
|
||||
# Set environment variables
|
||||
old_env = {}
|
||||
for key, value in self.model_config.get("env_vars", {}).items():
|
||||
old_env[key] = os.environ.get(key)
|
||||
os.environ[key] = value
|
||||
print(f"Setting env: {key}={value}")
|
||||
|
||||
try:
|
||||
result_tuple = self.runner.run_benchmark_for_model(
|
||||
model_path=self.model_config["model_path"],
|
||||
batch_sizes=self.batch_sizes,
|
||||
input_lens=self.input_lens,
|
||||
output_lens=self.output_lens,
|
||||
other_args=self.model_config["other_args"],
|
||||
variant=self.model_config["name"],
|
||||
extra_bench_args=["--trust-remote-code"],
|
||||
)
|
||||
results = result_tuple[0]
|
||||
success = result_tuple[1]
|
||||
|
||||
if results:
|
||||
self.runner.full_report += (
|
||||
generate_simple_markdown_report(results) + "\n"
|
||||
)
|
||||
|
||||
self.assertTrue(success, "Benchmark failed for Grok-1 INT4 on MI35x")
|
||||
finally:
|
||||
# Restore original environment
|
||||
for key, value in old_env.items():
|
||||
if value is None:
|
||||
os.environ.pop(key, None)
|
||||
else:
|
||||
os.environ[key] = value
|
||||
self.runner.write_final_report()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
126
test/registered/amd/perf/mi35x/test_grok2_perf_mi35x.py
Normal file
126
test/registered/amd/perf/mi35x/test_grok2_perf_mi35x.py
Normal file
@@ -0,0 +1,126 @@
|
||||
"""MI35x Nightly performance benchmark for Grok-2.
|
||||
|
||||
This test benchmarks Grok-2 with FP8 quantization on 8 GPUs.
|
||||
|
||||
Registry: nightly-perf-8-gpu-mi35x-grok2 suite
|
||||
"""
|
||||
|
||||
import os
|
||||
import unittest
|
||||
from typing import List
|
||||
|
||||
from sglang.test.ci.ci_register import register_amd_ci
|
||||
from sglang.test.nightly_bench_utils import BenchmarkResult
|
||||
from sglang.test.nightly_utils import NightlyBenchmarkRunner
|
||||
from sglang.test.test_utils import DEFAULT_URL_FOR_TEST, _parse_int_list_env
|
||||
|
||||
# Register for AMD CI - Grok-2 benchmark on MI35x (~25 min)
|
||||
register_amd_ci(est_time=1500, suite="nightly-perf-8-gpu-mi35x-grok2", nightly=True)
|
||||
|
||||
|
||||
def generate_simple_markdown_report(results: List[BenchmarkResult]) -> str:
|
||||
"""Generate a simplified markdown report without traces and cost columns."""
|
||||
model_header = results[0].model_path
|
||||
if results[0].run_name and results[0].run_name != "default":
|
||||
model_header += f" ({results[0].run_name})"
|
||||
|
||||
gpu_config = os.getenv("GPU_CONFIG", "MI35x")
|
||||
if gpu_config:
|
||||
model_header += f" [{gpu_config}]"
|
||||
|
||||
summary = f"### {model_header}\n"
|
||||
summary += "| batch size | input len | latency (s) | input throughput (tok/s) | output throughput (tok/s) | ITL (ms) |\n"
|
||||
summary += "| ---------- | --------- | ----------- | ------------------------ | ------------------------- | -------- |\n"
|
||||
|
||||
for result in results:
|
||||
itl = 1 / (result.output_throughput / result.batch_size) * 1000
|
||||
summary += f"| {result.batch_size} | {result.input_len} | {result.latency:.2f} | {result.input_throughput:.2f} | {result.output_throughput:.2f} | {itl:.2f} |\n"
|
||||
|
||||
return summary
|
||||
|
||||
|
||||
# Model and tokenizer paths can be overridden via environment variables
|
||||
GROK2_MODEL_PATH = os.environ.get("GROK2_MODEL_PATH", "xai-org/grok-2")
|
||||
GROK2_TOKENIZER_PATH = os.environ.get(
|
||||
"GROK2_TOKENIZER_PATH", "alvarobartt/grok-2-tokenizer"
|
||||
)
|
||||
PROFILE_DIR = "performance_profiles_grok2_mi35x"
|
||||
|
||||
|
||||
class TestGrok2PerfMI35x(unittest.TestCase):
|
||||
"""Test suite for Grok-2 performance benchmarks on MI35x."""
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.base_url = DEFAULT_URL_FOR_TEST
|
||||
cls.batch_sizes = [1, 1, 8, 16, 64]
|
||||
cls.input_lens = tuple(_parse_int_list_env("NIGHTLY_INPUT_LENS", "1024"))
|
||||
cls.output_lens = tuple(_parse_int_list_env("NIGHTLY_OUTPUT_LENS", "512"))
|
||||
|
||||
cls.model_config = {
|
||||
"name": "grok2-mi35x",
|
||||
"model_path": GROK2_MODEL_PATH,
|
||||
"other_args": [
|
||||
"--trust-remote-code",
|
||||
"--tp",
|
||||
"8",
|
||||
"--quantization",
|
||||
"fp8",
|
||||
"--mem-fraction-static",
|
||||
"0.85",
|
||||
"--tokenizer-path",
|
||||
GROK2_TOKENIZER_PATH,
|
||||
"--attention-backend",
|
||||
"aiter",
|
||||
],
|
||||
"env_vars": {
|
||||
"RCCL_MSCCL_ENABLE": "0",
|
||||
"SGLANG_USE_AITER": "1",
|
||||
"SGLANG_INT4_WEIGHT": "0",
|
||||
},
|
||||
}
|
||||
|
||||
cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url)
|
||||
cls.runner.setup_profile_directory()
|
||||
cls.runner.full_report = f"## {cls.__name__}\n"
|
||||
|
||||
def test_grok2_perf(self):
|
||||
"""Run Grok-2 performance benchmark on MI35x."""
|
||||
# Set environment variables
|
||||
old_env = {}
|
||||
for key, value in self.model_config.get("env_vars", {}).items():
|
||||
old_env[key] = os.environ.get(key)
|
||||
os.environ[key] = value
|
||||
print(f"Setting env: {key}={value}")
|
||||
|
||||
try:
|
||||
result_tuple = self.runner.run_benchmark_for_model(
|
||||
model_path=self.model_config["model_path"],
|
||||
batch_sizes=self.batch_sizes,
|
||||
input_lens=self.input_lens,
|
||||
output_lens=self.output_lens,
|
||||
other_args=self.model_config["other_args"],
|
||||
variant=self.model_config["name"],
|
||||
extra_bench_args=["--trust-remote-code"],
|
||||
)
|
||||
results = result_tuple[0]
|
||||
success = result_tuple[1]
|
||||
|
||||
if results:
|
||||
self.runner.full_report += (
|
||||
generate_simple_markdown_report(results) + "\n"
|
||||
)
|
||||
|
||||
self.assertTrue(success, "Benchmark failed for Grok-2 on MI35x")
|
||||
finally:
|
||||
# Restore original environment
|
||||
for key, value in old_env.items():
|
||||
if value is None:
|
||||
os.environ.pop(key, None)
|
||||
else:
|
||||
os.environ[key] = value
|
||||
self.runner.write_final_report()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
133
test/registered/amd/perf/test_grok1_fp8_perf.py
Normal file
133
test/registered/amd/perf/test_grok1_fp8_perf.py
Normal file
@@ -0,0 +1,133 @@
|
||||
"""Nightly performance benchmark for Grok-1 FP8.
|
||||
|
||||
This test benchmarks Grok-1 (314B MOE) with FP8 quantization on 8 GPUs.
|
||||
|
||||
Model paths can be configured via environment variables:
|
||||
- GROK1_MODEL_PATH: Path to Grok-1 model (default: lmzheng/grok-1)
|
||||
- GROK1_TOKENIZER_PATH: Path to Grok-1 tokenizer (default: Xenova/grok-1-tokenizer)
|
||||
|
||||
Example usage:
|
||||
python -m pytest test_grok1_fp8_perf.py -v
|
||||
"""
|
||||
|
||||
import os
|
||||
import unittest
|
||||
from typing import List
|
||||
|
||||
from sglang.test.ci.ci_register import register_amd_ci
|
||||
from sglang.test.nightly_bench_utils import BenchmarkResult
|
||||
from sglang.test.nightly_utils import NightlyBenchmarkRunner
|
||||
from sglang.test.test_utils import DEFAULT_URL_FOR_TEST, _parse_int_list_env
|
||||
|
||||
# Register for AMD CI - Grok-1 FP8 benchmark (~25 min)
|
||||
register_amd_ci(est_time=1500, suite="nightly-perf-8-gpu-grok1-fp8", nightly=True)
|
||||
|
||||
|
||||
def generate_simple_markdown_report(results: List[BenchmarkResult]) -> str:
|
||||
"""Generate a simplified markdown report without traces and cost columns."""
|
||||
model_header = results[0].model_path
|
||||
if results[0].run_name and results[0].run_name != "default":
|
||||
model_header += f" ({results[0].run_name})"
|
||||
|
||||
gpu_config = os.getenv("GPU_CONFIG", "")
|
||||
if gpu_config:
|
||||
model_header += f" [{gpu_config}]"
|
||||
|
||||
summary = f"### {model_header}\n"
|
||||
summary += "| batch size | input len | latency (s) | input throughput (tok/s) | output throughput (tok/s) | ITL (ms) |\n"
|
||||
summary += "| ---------- | --------- | ----------- | ------------------------ | ------------------------- | -------- |\n"
|
||||
|
||||
for result in results:
|
||||
itl = 1 / (result.output_throughput / result.batch_size) * 1000
|
||||
summary += f"| {result.batch_size} | {result.input_len} | {result.latency:.2f} | {result.input_throughput:.2f} | {result.output_throughput:.2f} | {itl:.2f} |\n"
|
||||
|
||||
return summary
|
||||
|
||||
|
||||
# Model and tokenizer paths can be overridden via environment variables
|
||||
GROK1_MODEL_PATH = os.environ.get("GROK1_MODEL_PATH", "lmzheng/grok-1")
|
||||
GROK1_TOKENIZER_PATH = os.environ.get("GROK1_TOKENIZER_PATH", "Xenova/grok-1-tokenizer")
|
||||
PROFILE_DIR = "performance_profiles_grok1_fp8"
|
||||
|
||||
|
||||
class TestNightlyGrok1FP8Performance(unittest.TestCase):
|
||||
"""Nightly performance benchmark for Grok-1 FP8.
|
||||
|
||||
Tests Grok-1 (314B MOE) with FP8 quantization on TP=8.
|
||||
Runtime: ~25 minutes
|
||||
"""
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.base_url = DEFAULT_URL_FOR_TEST
|
||||
cls.batch_sizes = [1, 1, 8, 16, 64]
|
||||
cls.input_lens = tuple(_parse_int_list_env("NIGHTLY_INPUT_LENS", "1024"))
|
||||
cls.output_lens = tuple(_parse_int_list_env("NIGHTLY_OUTPUT_LENS", "512"))
|
||||
|
||||
cls.model_config = {
|
||||
"name": "grok1-fp8",
|
||||
"model_path": GROK1_MODEL_PATH,
|
||||
"other_args": [
|
||||
"--trust-remote-code",
|
||||
"--tp",
|
||||
"8",
|
||||
"--quantization",
|
||||
"fp8",
|
||||
"--mem-fraction-static",
|
||||
"0.85",
|
||||
"--tokenizer-path",
|
||||
GROK1_TOKENIZER_PATH,
|
||||
"--attention-backend",
|
||||
"aiter",
|
||||
],
|
||||
"env_vars": {
|
||||
"RCCL_MSCCL_ENABLE": "0",
|
||||
"SGLANG_USE_AITER": "1",
|
||||
"SGLANG_INT4_WEIGHT": "0",
|
||||
},
|
||||
}
|
||||
|
||||
cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url)
|
||||
cls.runner.setup_profile_directory()
|
||||
cls.runner.full_report = f"## {cls.__name__}\n"
|
||||
|
||||
def test_bench_grok1_fp8(self):
|
||||
"""Run benchmark for Grok-1 FP8."""
|
||||
# Set environment variables
|
||||
old_env = {}
|
||||
for key, value in self.model_config.get("env_vars", {}).items():
|
||||
old_env[key] = os.environ.get(key)
|
||||
os.environ[key] = value
|
||||
print(f"Setting env: {key}={value}")
|
||||
|
||||
try:
|
||||
result_tuple = self.runner.run_benchmark_for_model(
|
||||
model_path=self.model_config["model_path"],
|
||||
batch_sizes=self.batch_sizes,
|
||||
input_lens=self.input_lens,
|
||||
output_lens=self.output_lens,
|
||||
other_args=self.model_config["other_args"],
|
||||
variant=self.model_config["name"],
|
||||
extra_bench_args=["--trust-remote-code"],
|
||||
)
|
||||
results = result_tuple[0]
|
||||
success = result_tuple[1]
|
||||
|
||||
if results:
|
||||
self.runner.full_report += (
|
||||
generate_simple_markdown_report(results) + "\n"
|
||||
)
|
||||
|
||||
self.assertTrue(success, "Benchmark failed for Grok-1 FP8")
|
||||
finally:
|
||||
# Restore original environment
|
||||
for key, value in old_env.items():
|
||||
if value is None:
|
||||
os.environ.pop(key, None)
|
||||
else:
|
||||
os.environ[key] = value
|
||||
self.runner.write_final_report()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
133
test/registered/amd/perf/test_grok1_int4_perf.py
Normal file
133
test/registered/amd/perf/test_grok1_int4_perf.py
Normal file
@@ -0,0 +1,133 @@
|
||||
"""Nightly performance benchmark for Grok-1 INT4 (W4A8KV8).
|
||||
|
||||
This test benchmarks Grok-1 (314B MOE) with INT4 weight quantization on 8 GPUs.
|
||||
|
||||
Model paths can be configured via environment variables:
|
||||
- GROK1_MODEL_PATH: Path to Grok-1 INT4 model (default: amd/grok-1-W4A8KV8)
|
||||
- GROK1_TOKENIZER_PATH: Path to Grok-1 tokenizer (default: Xenova/grok-1-tokenizer)
|
||||
|
||||
Example usage:
|
||||
python -m pytest test_grok1_int4_perf.py -v
|
||||
"""
|
||||
|
||||
import os
|
||||
import unittest
|
||||
from typing import List
|
||||
|
||||
from sglang.test.ci.ci_register import register_amd_ci
|
||||
from sglang.test.nightly_bench_utils import BenchmarkResult
|
||||
from sglang.test.nightly_utils import NightlyBenchmarkRunner
|
||||
from sglang.test.test_utils import DEFAULT_URL_FOR_TEST, _parse_int_list_env
|
||||
|
||||
# Register for AMD CI - Grok-1 INT4 benchmark (~25 min)
|
||||
register_amd_ci(est_time=1500, suite="nightly-perf-8-gpu-grok1-int4", nightly=True)
|
||||
|
||||
|
||||
def generate_simple_markdown_report(results: List[BenchmarkResult]) -> str:
|
||||
"""Generate a simplified markdown report without traces and cost columns."""
|
||||
model_header = results[0].model_path
|
||||
if results[0].run_name and results[0].run_name != "default":
|
||||
model_header += f" ({results[0].run_name})"
|
||||
|
||||
gpu_config = os.getenv("GPU_CONFIG", "")
|
||||
if gpu_config:
|
||||
model_header += f" [{gpu_config}]"
|
||||
|
||||
summary = f"### {model_header}\n"
|
||||
summary += "| batch size | input len | latency (s) | input throughput (tok/s) | output throughput (tok/s) | ITL (ms) |\n"
|
||||
summary += "| ---------- | --------- | ----------- | ------------------------ | ------------------------- | -------- |\n"
|
||||
|
||||
for result in results:
|
||||
itl = 1 / (result.output_throughput / result.batch_size) * 1000
|
||||
summary += f"| {result.batch_size} | {result.input_len} | {result.latency:.2f} | {result.input_throughput:.2f} | {result.output_throughput:.2f} | {itl:.2f} |\n"
|
||||
|
||||
return summary
|
||||
|
||||
|
||||
# Model and tokenizer paths can be overridden via environment variables
|
||||
GROK1_MODEL_PATH = os.environ.get("GROK1_MODEL_PATH", "amd/grok-1-W4A8KV8")
|
||||
GROK1_TOKENIZER_PATH = os.environ.get("GROK1_TOKENIZER_PATH", "Xenova/grok-1-tokenizer")
|
||||
PROFILE_DIR = "performance_profiles_grok1_int4"
|
||||
|
||||
|
||||
class TestNightlyGrok1INT4Performance(unittest.TestCase):
|
||||
"""Nightly performance benchmark for Grok-1 INT4 (W4A8KV8).
|
||||
|
||||
Tests Grok-1 (314B MOE) with INT4 weight quantization on TP=8.
|
||||
Runtime: ~25 minutes
|
||||
"""
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.base_url = DEFAULT_URL_FOR_TEST
|
||||
cls.batch_sizes = [1, 1, 8, 16, 64]
|
||||
cls.input_lens = tuple(_parse_int_list_env("NIGHTLY_INPUT_LENS", "1024"))
|
||||
cls.output_lens = tuple(_parse_int_list_env("NIGHTLY_OUTPUT_LENS", "512"))
|
||||
|
||||
cls.model_config = {
|
||||
"name": "grok1-int4",
|
||||
"model_path": GROK1_MODEL_PATH,
|
||||
"other_args": [
|
||||
"--trust-remote-code",
|
||||
"--tp",
|
||||
"8",
|
||||
"--quantization",
|
||||
"fp8",
|
||||
"--mem-fraction-static",
|
||||
"0.85",
|
||||
"--tokenizer-path",
|
||||
GROK1_TOKENIZER_PATH,
|
||||
"--attention-backend",
|
||||
"aiter",
|
||||
],
|
||||
"env_vars": {
|
||||
"RCCL_MSCCL_ENABLE": "0",
|
||||
"SGLANG_USE_AITER": "1",
|
||||
"SGLANG_INT4_WEIGHT": "1",
|
||||
},
|
||||
}
|
||||
|
||||
cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url)
|
||||
cls.runner.setup_profile_directory()
|
||||
cls.runner.full_report = f"## {cls.__name__}\n"
|
||||
|
||||
def test_bench_grok1_int4(self):
|
||||
"""Run benchmark for Grok-1 INT4."""
|
||||
# Set environment variables
|
||||
old_env = {}
|
||||
for key, value in self.model_config.get("env_vars", {}).items():
|
||||
old_env[key] = os.environ.get(key)
|
||||
os.environ[key] = value
|
||||
print(f"Setting env: {key}={value}")
|
||||
|
||||
try:
|
||||
result_tuple = self.runner.run_benchmark_for_model(
|
||||
model_path=self.model_config["model_path"],
|
||||
batch_sizes=self.batch_sizes,
|
||||
input_lens=self.input_lens,
|
||||
output_lens=self.output_lens,
|
||||
other_args=self.model_config["other_args"],
|
||||
variant=self.model_config["name"],
|
||||
extra_bench_args=["--trust-remote-code"],
|
||||
)
|
||||
results = result_tuple[0]
|
||||
success = result_tuple[1]
|
||||
|
||||
if results:
|
||||
self.runner.full_report += (
|
||||
generate_simple_markdown_report(results) + "\n"
|
||||
)
|
||||
|
||||
self.assertTrue(success, "Benchmark failed for Grok-1 INT4")
|
||||
finally:
|
||||
# Restore original environment
|
||||
for key, value in old_env.items():
|
||||
if value is None:
|
||||
os.environ.pop(key, None)
|
||||
else:
|
||||
os.environ[key] = value
|
||||
self.runner.write_final_report()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
135
test/registered/amd/perf/test_grok2_perf.py
Normal file
135
test/registered/amd/perf/test_grok2_perf.py
Normal file
@@ -0,0 +1,135 @@
|
||||
"""Nightly performance benchmark for Grok-2.
|
||||
|
||||
This test benchmarks Grok-2 with FP8 quantization on 8 GPUs.
|
||||
|
||||
Model paths can be configured via environment variables:
|
||||
- GROK2_MODEL_PATH: Path to Grok-2 model (default: xai-org/grok-2)
|
||||
- GROK2_TOKENIZER_PATH: Path to Grok-2 tokenizer (default: alvarobartt/grok-2-tokenizer)
|
||||
|
||||
Example usage:
|
||||
python -m pytest test_grok2_perf.py -v
|
||||
"""
|
||||
|
||||
import os
|
||||
import unittest
|
||||
from typing import List
|
||||
|
||||
from sglang.test.ci.ci_register import register_amd_ci
|
||||
from sglang.test.nightly_bench_utils import BenchmarkResult
|
||||
from sglang.test.nightly_utils import NightlyBenchmarkRunner
|
||||
from sglang.test.test_utils import DEFAULT_URL_FOR_TEST, _parse_int_list_env
|
||||
|
||||
# Register for AMD CI - Grok-2 benchmark (~25 min)
|
||||
register_amd_ci(est_time=1500, suite="nightly-perf-8-gpu-grok2", nightly=True)
|
||||
|
||||
|
||||
def generate_simple_markdown_report(results: List[BenchmarkResult]) -> str:
|
||||
"""Generate a simplified markdown report without traces and cost columns."""
|
||||
model_header = results[0].model_path
|
||||
if results[0].run_name and results[0].run_name != "default":
|
||||
model_header += f" ({results[0].run_name})"
|
||||
|
||||
gpu_config = os.getenv("GPU_CONFIG", "")
|
||||
if gpu_config:
|
||||
model_header += f" [{gpu_config}]"
|
||||
|
||||
summary = f"### {model_header}\n"
|
||||
summary += "| batch size | input len | latency (s) | input throughput (tok/s) | output throughput (tok/s) | ITL (ms) |\n"
|
||||
summary += "| ---------- | --------- | ----------- | ------------------------ | ------------------------- | -------- |\n"
|
||||
|
||||
for result in results:
|
||||
itl = 1 / (result.output_throughput / result.batch_size) * 1000
|
||||
summary += f"| {result.batch_size} | {result.input_len} | {result.latency:.2f} | {result.input_throughput:.2f} | {result.output_throughput:.2f} | {itl:.2f} |\n"
|
||||
|
||||
return summary
|
||||
|
||||
|
||||
# Model and tokenizer paths can be overridden via environment variables
|
||||
GROK2_MODEL_PATH = os.environ.get("GROK2_MODEL_PATH", "xai-org/grok-2")
|
||||
GROK2_TOKENIZER_PATH = os.environ.get(
|
||||
"GROK2_TOKENIZER_PATH", "alvarobartt/grok-2-tokenizer"
|
||||
)
|
||||
PROFILE_DIR = "performance_profiles_grok2"
|
||||
|
||||
|
||||
class TestNightlyGrok2Performance(unittest.TestCase):
|
||||
"""Nightly performance benchmark for Grok-2.
|
||||
|
||||
Tests Grok-2 with FP8 quantization on TP=8.
|
||||
Runtime: ~25 minutes
|
||||
"""
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.base_url = DEFAULT_URL_FOR_TEST
|
||||
cls.batch_sizes = [1, 1, 8, 16, 64]
|
||||
cls.input_lens = tuple(_parse_int_list_env("NIGHTLY_INPUT_LENS", "1024"))
|
||||
cls.output_lens = tuple(_parse_int_list_env("NIGHTLY_OUTPUT_LENS", "512"))
|
||||
|
||||
cls.model_config = {
|
||||
"name": "grok2",
|
||||
"model_path": GROK2_MODEL_PATH,
|
||||
"other_args": [
|
||||
"--trust-remote-code",
|
||||
"--tp",
|
||||
"8",
|
||||
"--quantization",
|
||||
"fp8",
|
||||
"--mem-fraction-static",
|
||||
"0.85",
|
||||
"--tokenizer-path",
|
||||
GROK2_TOKENIZER_PATH,
|
||||
"--attention-backend",
|
||||
"aiter",
|
||||
],
|
||||
"env_vars": {
|
||||
"RCCL_MSCCL_ENABLE": "0",
|
||||
"SGLANG_USE_AITER": "1",
|
||||
"SGLANG_INT4_WEIGHT": "0",
|
||||
},
|
||||
}
|
||||
|
||||
cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url)
|
||||
cls.runner.setup_profile_directory()
|
||||
cls.runner.full_report = f"## {cls.__name__}\n"
|
||||
|
||||
def test_bench_grok2(self):
|
||||
"""Run benchmark for Grok-2."""
|
||||
# Set environment variables
|
||||
old_env = {}
|
||||
for key, value in self.model_config.get("env_vars", {}).items():
|
||||
old_env[key] = os.environ.get(key)
|
||||
os.environ[key] = value
|
||||
print(f"Setting env: {key}={value}")
|
||||
|
||||
try:
|
||||
result_tuple = self.runner.run_benchmark_for_model(
|
||||
model_path=self.model_config["model_path"],
|
||||
batch_sizes=self.batch_sizes,
|
||||
input_lens=self.input_lens,
|
||||
output_lens=self.output_lens,
|
||||
other_args=self.model_config["other_args"],
|
||||
variant=self.model_config["name"],
|
||||
extra_bench_args=["--trust-remote-code"],
|
||||
)
|
||||
results = result_tuple[0]
|
||||
success = result_tuple[1]
|
||||
|
||||
if results:
|
||||
self.runner.full_report += (
|
||||
generate_simple_markdown_report(results) + "\n"
|
||||
)
|
||||
|
||||
self.assertTrue(success, "Benchmark failed for Grok-2")
|
||||
finally:
|
||||
# Restore original environment
|
||||
for key, value in old_env.items():
|
||||
if value is None:
|
||||
os.environ.pop(key, None)
|
||||
else:
|
||||
os.environ[key] = value
|
||||
self.runner.write_final_report()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -1,178 +0,0 @@
|
||||
"""Nightly performance benchmark for Grok models (Grok-1 and Grok-2).
|
||||
|
||||
This test benchmarks both Grok-1 and Grok-2 models with FP8 quantization on 8 GPUs.
|
||||
|
||||
Model paths can be configured via environment variables:
|
||||
- GROK1_MODEL_PATH: Path to Grok-1 model (default: amd/grok-1-W4A8KV8)
|
||||
- GROK1_TOKENIZER_PATH: Path to Grok-1 tokenizer (default: Xenova/grok-1-tokenizer)
|
||||
- GROK2_MODEL_PATH: Path to Grok-2 model (default: xai-org/grok-2)
|
||||
- GROK2_TOKENIZER_PATH: Path to Grok-2 tokenizer (default: alvarobartt/grok-2-tokenizer)
|
||||
|
||||
Example usage:
|
||||
python -m pytest test_grok_perf.py -v
|
||||
"""
|
||||
|
||||
import os
|
||||
import unittest
|
||||
from typing import List
|
||||
|
||||
from sglang.test.ci.ci_register import register_amd_ci
|
||||
from sglang.test.nightly_bench_utils import BenchmarkResult
|
||||
from sglang.test.nightly_utils import NightlyBenchmarkRunner
|
||||
from sglang.test.test_utils import DEFAULT_URL_FOR_TEST, _parse_int_list_env
|
||||
|
||||
# Register for AMD CI - combined Grok-1 + Grok-2 benchmark (~60 min)
|
||||
register_amd_ci(est_time=3600, suite="nightly-perf-8-gpu-grok", nightly=True)
|
||||
|
||||
|
||||
def generate_simple_markdown_report(results: List[BenchmarkResult]) -> str:
|
||||
"""Generate a simplified markdown report without traces and cost columns."""
|
||||
model_header = results[0].model_path
|
||||
if results[0].run_name and results[0].run_name != "default":
|
||||
model_header += f" ({results[0].run_name})"
|
||||
|
||||
gpu_config = os.getenv("GPU_CONFIG", "")
|
||||
if gpu_config:
|
||||
model_header += f" [{gpu_config}]"
|
||||
|
||||
summary = f"### {model_header}\n"
|
||||
summary += "| batch size | input len | latency (s) | input throughput (tok/s) | output throughput (tok/s) | ITL (ms) |\n"
|
||||
summary += "| ---------- | --------- | ----------- | ------------------------ | ------------------------- | -------- |\n"
|
||||
|
||||
for result in results:
|
||||
itl = 1 / (result.output_throughput / result.batch_size) * 1000
|
||||
summary += f"| {result.batch_size} | {result.input_len} | {result.latency:.2f} | {result.input_throughput:.2f} | {result.output_throughput:.2f} | {itl:.2f} |\n"
|
||||
|
||||
return summary
|
||||
|
||||
|
||||
# Model and tokenizer paths can be overridden via environment variables
|
||||
GROK1_MODEL_PATH = os.environ.get("GROK1_MODEL_PATH", "amd/grok-1-W4A8KV8")
|
||||
GROK1_TOKENIZER_PATH = os.environ.get("GROK1_TOKENIZER_PATH", "Xenova/grok-1-tokenizer")
|
||||
GROK2_MODEL_PATH = os.environ.get("GROK2_MODEL_PATH", "xai-org/grok-2")
|
||||
GROK2_TOKENIZER_PATH = os.environ.get(
|
||||
"GROK2_TOKENIZER_PATH", "alvarobartt/grok-2-tokenizer"
|
||||
)
|
||||
PROFILE_DIR = "performance_profiles_grok"
|
||||
|
||||
|
||||
class TestNightlyGrokPerformance(unittest.TestCase):
|
||||
"""Nightly performance benchmark for Grok models (Grok-1 and Grok-2).
|
||||
|
||||
Tests both Grok-1 (314B MOE) and Grok-2 models with FP8 quantization on TP=8.
|
||||
Combined runtime: ~43 minutes (Grok-1: ~23min, Grok-2: ~20min)
|
||||
"""
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.base_url = DEFAULT_URL_FOR_TEST
|
||||
cls.batch_sizes = [1, 1, 8, 16, 64]
|
||||
cls.input_lens = tuple(_parse_int_list_env("NIGHTLY_INPUT_LENS", "1024"))
|
||||
cls.output_lens = tuple(_parse_int_list_env("NIGHTLY_OUTPUT_LENS", "512"))
|
||||
|
||||
# Define model configurations for both Grok-1 and Grok-2
|
||||
cls.models = [
|
||||
{
|
||||
"name": "grok1",
|
||||
"model_path": GROK1_MODEL_PATH,
|
||||
"other_args": [
|
||||
"--trust-remote-code",
|
||||
"--tp",
|
||||
"8",
|
||||
"--quantization",
|
||||
"fp8",
|
||||
"--mem-fraction-static",
|
||||
"0.85",
|
||||
"--tokenizer-path",
|
||||
GROK1_TOKENIZER_PATH,
|
||||
"--attention-backend",
|
||||
"aiter",
|
||||
],
|
||||
"env_vars": {
|
||||
"RCCL_MSCCL_ENABLE": "0",
|
||||
"SGLANG_USE_AITER": "1",
|
||||
"SGLANG_INT4_WEIGHT": "1",
|
||||
},
|
||||
},
|
||||
{
|
||||
"name": "grok2",
|
||||
"model_path": GROK2_MODEL_PATH,
|
||||
"other_args": [
|
||||
"--trust-remote-code",
|
||||
"--tp",
|
||||
"8",
|
||||
"--quantization",
|
||||
"fp8",
|
||||
"--mem-fraction-static",
|
||||
"0.85",
|
||||
"--tokenizer-path",
|
||||
GROK2_TOKENIZER_PATH,
|
||||
"--attention-backend",
|
||||
"aiter",
|
||||
],
|
||||
"env_vars": {
|
||||
"RCCL_MSCCL_ENABLE": "0",
|
||||
"SGLANG_USE_AITER": "1",
|
||||
"SGLANG_INT4_WEIGHT": "0",
|
||||
},
|
||||
},
|
||||
]
|
||||
|
||||
cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url)
|
||||
cls.runner.setup_profile_directory()
|
||||
# Override full_report to remove traces help text
|
||||
cls.runner.full_report = f"## {cls.__name__}\n"
|
||||
|
||||
def test_bench_one_batch(self):
|
||||
"""Run benchmark across all Grok models."""
|
||||
failed_models = []
|
||||
|
||||
try:
|
||||
for model_config in self.models:
|
||||
with self.subTest(model=model_config["name"]):
|
||||
# Set environment variables for this model
|
||||
old_env = {}
|
||||
for key, value in model_config.get("env_vars", {}).items():
|
||||
old_env[key] = os.environ.get(key)
|
||||
os.environ[key] = value
|
||||
print(f"Setting env: {key}={value}")
|
||||
|
||||
try:
|
||||
result_tuple = self.runner.run_benchmark_for_model(
|
||||
model_path=model_config["model_path"],
|
||||
batch_sizes=self.batch_sizes,
|
||||
input_lens=self.input_lens,
|
||||
output_lens=self.output_lens,
|
||||
other_args=model_config["other_args"],
|
||||
variant=model_config["name"],
|
||||
extra_bench_args=["--trust-remote-code"],
|
||||
)
|
||||
results = result_tuple[0]
|
||||
success = result_tuple[1]
|
||||
|
||||
if not success:
|
||||
failed_models.append(model_config["name"])
|
||||
|
||||
# Use simplified report format without traces
|
||||
if results:
|
||||
self.runner.full_report += (
|
||||
generate_simple_markdown_report(results) + "\n"
|
||||
)
|
||||
finally:
|
||||
# Restore original environment
|
||||
for key, value in old_env.items():
|
||||
if value is None:
|
||||
os.environ.pop(key, None)
|
||||
else:
|
||||
os.environ[key] = value
|
||||
finally:
|
||||
self.runner.write_final_report()
|
||||
|
||||
if failed_models:
|
||||
raise AssertionError(
|
||||
f"Benchmark failed for the following models: {', '.join(failed_models)}"
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -8,7 +8,7 @@ from pathlib import Path
|
||||
|
||||
from sglang.bench_serving import parse_custom_headers, run_benchmark
|
||||
from sglang.srt.utils import kill_process_tree
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
|
||||
from sglang.test.test_utils import (
|
||||
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||
DEFAULT_URL_FOR_TEST,
|
||||
@@ -18,6 +18,7 @@ from sglang.test.test_utils import (
|
||||
)
|
||||
|
||||
register_cuda_ci(est_time=300, suite="nightly-1-gpu", nightly=True)
|
||||
register_amd_ci(est_time=300, suite="nightly-amd-1-gpu", nightly=True)
|
||||
|
||||
MODEL = "Qwen/Qwen3-0.6B"
|
||||
NUM_CONVERSATIONS, NUM_TURNS = 4, 3
|
||||
|
||||
@@ -5,12 +5,15 @@ import unittest
|
||||
import torch
|
||||
|
||||
from sglang.srt.batch_invariant_ops import batch_invariant_ops
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
|
||||
register_cuda_ci(est_time=10, suite="nightly-1-gpu", nightly=True)
|
||||
from sglang.srt.batch_invariant_ops.batch_invariant_ops import set_batch_invariant_mode
|
||||
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
# Note: MI300 (gfx942) has 64KB shared memory limit but kernel needs 66KB
|
||||
# MI35x (gfx950/CDNA4) may have different limits - testing on MI35x only
|
||||
register_cuda_ci(est_time=10, suite="nightly-1-gpu", nightly=True)
|
||||
register_amd_ci(est_time=10, suite="nightly-amd-1-gpu-mi35x", nightly=True)
|
||||
|
||||
device_type = getattr(torch.accelerator.current_accelerator(), "type", "cpu")
|
||||
torch.set_default_device(device_type)
|
||||
|
||||
|
||||
@@ -13,6 +13,7 @@ from sglang.test.test_utils import (
|
||||
popen_launch_server,
|
||||
)
|
||||
|
||||
# Note: AMD registration removed - test_cpp_radix_cache fails on AMD due to C++ radix tree issues
|
||||
register_cuda_ci(est_time=60, suite="nightly-1-gpu", nightly=True)
|
||||
|
||||
|
||||
|
||||
@@ -9,7 +9,7 @@ import requests
|
||||
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.utils import kill_process_tree
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
|
||||
from sglang.test.test_utils import (
|
||||
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||
DEFAULT_URL_FOR_TEST,
|
||||
@@ -18,6 +18,7 @@ from sglang.test.test_utils import (
|
||||
)
|
||||
|
||||
register_cuda_ci(est_time=40, suite="nightly-1-gpu", nightly=True)
|
||||
register_amd_ci(est_time=40, suite="nightly-amd-1-gpu", nightly=True)
|
||||
|
||||
|
||||
class TestCrashDump(CustomTestCase):
|
||||
|
||||
@@ -9,10 +9,11 @@ import torch
|
||||
import torch.distributed as dist
|
||||
import torch.multiprocessing as mp
|
||||
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
register_cuda_ci(est_time=30, suite="nightly-2-gpu", nightly=True)
|
||||
register_amd_ci(est_time=60, suite="nightly-amd", nightly=True)
|
||||
|
||||
|
||||
class TestDumperPureFunctions(CustomTestCase):
|
||||
|
||||
@@ -5,7 +5,7 @@ import requests
|
||||
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.utils import kill_process_tree
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
|
||||
from sglang.test.test_utils import (
|
||||
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||
DEFAULT_URL_FOR_TEST,
|
||||
@@ -14,6 +14,7 @@ from sglang.test.test_utils import (
|
||||
)
|
||||
|
||||
register_cuda_ci(est_time=120, suite="nightly-1-gpu", nightly=True)
|
||||
register_amd_ci(est_time=120, suite="nightly-amd-1-gpu", nightly=True)
|
||||
|
||||
|
||||
class BaseTestSoftWatchdog:
|
||||
|
||||
@@ -20,9 +20,10 @@ Tests LRU and FIFO eviction behavior.
|
||||
import unittest
|
||||
|
||||
from sglang.srt.lora.eviction_policy import get_eviction_policy
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
|
||||
|
||||
register_cuda_ci(est_time=200, suite="nightly-1-gpu", nightly=True)
|
||||
register_amd_ci(est_time=200, suite="nightly-amd-1-gpu", nightly=True)
|
||||
|
||||
|
||||
class TestLoRAEvictionPolicy(unittest.TestCase):
|
||||
|
||||
@@ -9,10 +9,11 @@ import unittest
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from sglang.srt.entrypoints.openai.serving_base import OpenAIServingBase
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
from sglang.srt.server_args import ServerArgs
|
||||
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
|
||||
|
||||
register_cuda_ci(est_time=30, suite="nightly-1-gpu", nightly=True)
|
||||
from sglang.srt.server_args import ServerArgs
|
||||
register_amd_ci(est_time=30, suite="nightly-amd-1-gpu", nightly=True)
|
||||
|
||||
|
||||
class MockTokenizerManager:
|
||||
|
||||
@@ -17,9 +17,10 @@ import unittest
|
||||
|
||||
import openai
|
||||
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
|
||||
|
||||
register_cuda_ci(est_time=150, suite="nightly-1-gpu", nightly=True)
|
||||
register_amd_ci(est_time=150, suite="nightly-amd-1-gpu", nightly=True)
|
||||
|
||||
from sglang.srt.utils import kill_process_tree
|
||||
from sglang.test.test_utils import (
|
||||
|
||||
@@ -17,12 +17,12 @@ import unittest
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
|
||||
from sglang.test.lora_utils import CI_MULTI_LORA_MODELS, run_lora_test_one_by_one
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
register_cuda_ci(est_time=200, suite="nightly-1-gpu", nightly=True)
|
||||
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
register_amd_ci(est_time=200, suite="nightly-amd-1-gpu", nightly=True)
|
||||
|
||||
PROMPTS = [
|
||||
"AI is a field of computer science focused on",
|
||||
|
||||
@@ -6,7 +6,7 @@ import unittest
|
||||
import aiohttp
|
||||
|
||||
from sglang.srt.utils import kill_process_tree
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
|
||||
from sglang.test.test_utils import (
|
||||
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||
DEFAULT_URL_FOR_TEST,
|
||||
@@ -17,6 +17,7 @@ from sglang.test.test_utils import (
|
||||
)
|
||||
|
||||
register_cuda_ci(est_time=120, suite="nightly-1-gpu", nightly=True)
|
||||
register_amd_ci(est_time=120, suite="nightly-amd-1-gpu", nightly=True)
|
||||
|
||||
|
||||
class TestRoutingKeyScheduling(CustomTestCase):
|
||||
|
||||
@@ -27,6 +27,7 @@ from sglang.test.test_utils import (
|
||||
popen_launch_server,
|
||||
)
|
||||
|
||||
# Note: AMD registration removed - test_model_file_verifier fails on AMD
|
||||
register_cuda_ci(est_time=120, suite="nightly-1-gpu", nightly=True)
|
||||
|
||||
MODEL_NAME = "Qwen/Qwen3-0.6B"
|
||||
|
||||
@@ -8,7 +8,7 @@ from pathlib import Path
|
||||
import requests
|
||||
|
||||
from sglang.srt.utils import kill_process_tree
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
|
||||
from sglang.test.test_utils import (
|
||||
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||
DEFAULT_URL_FOR_TEST,
|
||||
@@ -17,6 +17,7 @@ from sglang.test.test_utils import (
|
||||
)
|
||||
|
||||
register_cuda_ci(est_time=120, suite="nightly-1-gpu", nightly=True)
|
||||
register_amd_ci(est_time=120, suite="nightly-amd-1-gpu", nightly=True)
|
||||
|
||||
TEST_ROUTING_KEY = "test-routing-key-12345"
|
||||
|
||||
|
||||
@@ -9,7 +9,7 @@ from pathlib import Path
|
||||
import requests
|
||||
|
||||
from sglang.srt.utils import kill_process_tree
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
|
||||
from sglang.test.test_utils import (
|
||||
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||
DEFAULT_URL_FOR_TEST,
|
||||
@@ -18,6 +18,7 @@ from sglang.test.test_utils import (
|
||||
)
|
||||
|
||||
register_cuda_ci(est_time=120, suite="nightly-1-gpu", nightly=True)
|
||||
register_amd_ci(est_time=120, suite="nightly-amd-1-gpu", nightly=True)
|
||||
|
||||
|
||||
class TestSchedulerStatusLogger(CustomTestCase):
|
||||
|
||||
Reference in New Issue
Block a user