diff --git a/.github/workflows/nightly-test-amd.yml b/.github/workflows/nightly-test-amd.yml index e4250fbc1..155353975 100644 --- a/.github/workflows/nightly-test-amd.yml +++ b/.github/workflows/nightly-test-amd.yml @@ -17,22 +17,24 @@ on: default: 'all' options: - 'all' - - 'nightly-test-2-gpu' - - 'nightly-test-2-gpu-vlm' - - 'nightly-test-8-gpu-gpt-oss' - - 'nightly-test-8-gpu-grok' - - 'nightly-test-8-gpu-deepseek-r1' - - 'nightly-perf-8-gpu-grok' - - 'nightly-perf-8-gpu-deepseek-v3' - - 'nightly-perf-8-gpu-deepseek-v31' + # MI30x Unit Tests + - 'nightly-test-1-gpu-unit' + # MI30x Accuracy Tests (GSM8K / MMMU) + - 'nightly-accuracy-2-gpu' + - 'nightly-accuracy-2-gpu-vlm' + - 'nightly-accuracy-8-gpu' + - 'nightly-accuracy-8-gpu-deepseek-r1' + # MI30x Accuracy + Performance Tests (combined) + - 'nightly-8-gpu-grok1-int4' + - 'nightly-8-gpu-grok2' + - 'nightly-8-gpu-deepseek-v31' # MI35x jobs - - 'nightly-test-2-gpu-mi35x' - - 'nightly-test-2-gpu-vlm-mi35x' - - 'nightly-test-8-gpu-mi35x-gpt-oss' - - 'nightly-test-8-gpu-mi35x-grok' - - 'nightly-test-8-gpu-mi35x-deepseek-r1' - - 'nightly-perf-8-gpu-mi35x-grok' - - 'nightly-perf-8-gpu-mi35x-deepseek-r1-mxfp4' + - 'nightly-test-1-gpu-mi35x' + - 'nightly-accuracy-8-gpu-mi35x' + - 'nightly-accuracy-8-gpu-mi35x-deepseek-r1' + - 'nightly-8-gpu-mi35x-grok1-int4' + - 'nightly-8-gpu-mi35x-grok2' + - 'nightly-8-gpu-mi35x-deepseek-r1-mxfp4' workflow_call: inputs: ref: @@ -51,9 +53,40 @@ concurrency: cancel-in-progress: ${{ github.event_name != 'workflow_call' }} jobs: - # 2-GPU tests (TP=2) - nightly-test-2-gpu: - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-2-gpu') + # ============================================== MI30x Unit Tests ============================================== + # 1-GPU Unit Tests - LoRA, debug utils, scheduler, etc. (MI30x only) + nightly-test-1-gpu-unit: + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-1-gpu-unit') + runs-on: linux-mi325-gpu-1 + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref || github.ref }} + + - name: Setup docker + run: | + touch github_summary.md + bash scripts/ci/amd_ci_start_container.sh + env: + GITHUB_WORKSPACE: ${{ github.workspace }} + + - name: Install dependencies + run: bash scripts/ci/amd_ci_install_dependency.sh + + - name: Nightly Unit Test (1-GPU) + timeout-minutes: 60 + run: | + bash scripts/ci/amd_ci_exec.sh -w /sglang-checkout/test \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + python3 run_suite.py --hw amd --suite nightly-amd-1-gpu --nightly --timeout-per-file 600 --continue-on-error || TEST_EXIT_CODE=$? + echo "$(> $GITHUB_STEP_SUMMARY || true + exit ${TEST_EXIT_CODE:-0} + + # ============================================== MI30x Accuracy Tests ============================================== + # 2-GPU Accuracy Tests - GSM8K eval (MI30x only) + nightly-accuracy-2-gpu: + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-accuracy-2-gpu') runs-on: linux-mi325-gpu-2 steps: - name: Checkout code @@ -79,9 +112,9 @@ jobs: echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} - # 2-GPU VLM tests - Vision-Language Models MMMU evaluation - nightly-test-2-gpu-vlm: - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-2-gpu-vlm') + # 2-GPU VLM Accuracy Tests - Vision-Language Models MMMU evaluation + nightly-accuracy-2-gpu-vlm: + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-accuracy-2-gpu-vlm') runs-on: linux-mi325-gpu-2 steps: - name: Checkout code @@ -99,18 +132,18 @@ jobs: - name: Install dependencies run: bash scripts/ci/amd_ci_install_dependency.sh - - name: Nightly Test (2-GPU VLM MMMU) + - name: Nightly Accuracy Test (2-GPU VLM MMMU) timeout-minutes: 180 run: | bash scripts/ci/amd_ci_exec.sh -w /sglang-checkout/test \ -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-amd-vlm --nightly --timeout-per-file 7200 || TEST_EXIT_CODE=$? + python3 run_suite.py --hw amd --suite nightly-amd-accuracy-2-gpu-vlm --nightly --timeout-per-file 7200 || TEST_EXIT_CODE=$? echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} - # 8-GPU tests (TP=8) - GPT-OSS models - nightly-test-8-gpu-gpt-oss: - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-8-gpu-gpt-oss') + # 8-GPU Accuracy Tests - GPT-OSS, Grok1-FP8 (accuracy only) + nightly-accuracy-8-gpu: + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-accuracy-8-gpu') runs-on: linux-mi325-gpu-8 steps: - name: Checkout code @@ -128,103 +161,28 @@ jobs: - name: Install dependencies run: bash scripts/ci/amd_ci_install_dependency.sh - - name: Nightly Test (8-GPU GPT-OSS) + - name: Accuracy Test (8-GPU GPT-OSS) + timeout-minutes: 180 run: | bash scripts/ci/amd_ci_exec.sh -w /sglang-checkout/test \ - -e AMD_TEST_MODEL_GROUP=gpt-oss \ -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-amd-8-gpu --nightly --timeout-per-file 7200 || TEST_EXIT_CODE=$? + python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-gpt-oss --nightly --timeout-per-file 7200 || TEST_EXIT_CODE=$? echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} - # 8-GPU tests (TP=8) - GROK models (GROK1-FP8, GROK1-IN4, GROK2.5) - nightly-test-8-gpu-grok: - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-8-gpu-grok') - runs-on: linux-mi325-gpu-8 - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref || github.ref }} - - - name: Setup docker - run: | - touch github_summary.md - bash scripts/ci/amd_ci_start_container.sh - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - - - name: Install dependencies - run: bash scripts/ci/amd_ci_install_dependency.sh - - - name: Nightly Test (8-GPU GROK) - run: | - bash scripts/ci/amd_ci_exec.sh -w /sglang-checkout/test \ - -e AMD_TEST_MODEL_GROUP=grok \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-amd-8-gpu --nightly --timeout-per-file 7200 || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - # 8-GPU tests (TP=8) - DeepSeek-R1 all variants (basic, MTP, DP, TC) - nightly-test-8-gpu-deepseek-r1: - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-8-gpu-deepseek-r1') - runs-on: linux-mi325-gpu-8 - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref || github.ref }} - - - name: Setup docker - run: | - touch github_summary.md - bash scripts/ci/amd_ci_start_container.sh - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - - - name: Install dependencies - run: bash scripts/ci/amd_ci_install_dependency.sh - - - name: Nightly Test (8-GPU DeepSeek-R1 all variants) - run: | - bash scripts/ci/amd_ci_exec.sh -w /sglang-checkout/test \ - -e AMD_TEST_MODEL_GROUP=deepseek-r1-all \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-amd-8-gpu --nightly --timeout-per-file 7200 || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - # 8-GPU Performance Tests (TP=8) - Grok (Grok-1 + Grok-2) performance benchmarks - nightly-perf-8-gpu-grok: - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-perf-8-gpu-grok') - runs-on: linux-mi325-gpu-8 - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref || github.ref }} - - - name: Setup docker - run: | - touch github_summary.md - bash scripts/ci/amd_ci_start_container.sh - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - - - name: Install dependencies - run: bash scripts/ci/amd_ci_install_dependency.sh - - - name: Nightly Perf Test (8-GPU Grok-1 + Grok-2) + - name: Accuracy Test (8-GPU Grok1-FP8) timeout-minutes: 60 run: | - bash scripts/ci/amd_ci_exec.sh -w /sglang-checkout/test -e RCCL_MSCCL_ENABLE=0 -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" python3 registered/amd/test_grok_perf.py || TEST_EXIT_CODE=$? + bash scripts/ci/amd_ci_exec.sh -w /sglang-checkout/test \ + -e RCCL_MSCCL_ENABLE=0 \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-grok1-fp8 --nightly --timeout-per-file 3600 || TEST_EXIT_CODE=$? echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} - # 8-GPU Performance Tests (TP=8) - DeepSeek-V3 performance benchmarks - nightly-perf-8-gpu-deepseek-v3: - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-perf-8-gpu-deepseek-v3') + # 8-GPU DeepSeek-R1 Accuracy Test (separate job due to long loading time) + nightly-accuracy-8-gpu-deepseek-r1: + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-accuracy-8-gpu-deepseek-r1') runs-on: linux-mi325-gpu-8 steps: - name: Checkout code @@ -242,16 +200,19 @@ jobs: - name: Install dependencies run: bash scripts/ci/amd_ci_install_dependency.sh - - name: Nightly Perf Test (8-GPU DeepSeek-V3) - timeout-minutes: 300 + - name: Accuracy Test (8-GPU DeepSeek-R1) + timeout-minutes: 240 run: | - bash scripts/ci/amd_ci_exec.sh -w /sglang-checkout/test -e SGLANG_USE_ROCM700A=1 -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" python3 registered/amd/test_deepseek_v3_perf.py || TEST_EXIT_CODE=$? + bash scripts/ci/amd_ci_exec.sh -w /sglang-checkout/test \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-deepseek-r1 --nightly --timeout-per-file 7200 || TEST_EXIT_CODE=$? echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} - # 8-GPU Performance Tests (TP=8) - DeepSeek-V3.1 performance benchmarks - nightly-perf-8-gpu-deepseek-v31: - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-perf-8-gpu-deepseek-v31') + # ============================================== MI30x Combined Accuracy + Performance Tests ============================================== + # 8-GPU Grok1-INT4 (Accuracy + Performance combined) + nightly-8-gpu-grok1-int4: + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-8-gpu-grok1-int4') runs-on: linux-mi325-gpu-8 steps: - name: Checkout code @@ -269,18 +230,120 @@ jobs: - name: Install dependencies run: bash scripts/ci/amd_ci_install_dependency.sh - - name: Nightly Perf Test (8-GPU DeepSeek-V3.1) - timeout-minutes: 300 + - name: Accuracy Test (8-GPU Grok1-INT4) + timeout-minutes: 60 run: | - bash scripts/ci/amd_ci_exec.sh -w /sglang-checkout/test -e SGLANG_USE_ROCM700A=1 -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" python3 registered/amd/test_deepseek_v31_perf.py || TEST_EXIT_CODE=$? + > github_summary.md # Clear summary file + bash scripts/ci/amd_ci_exec.sh -w /sglang-checkout/test \ + -e RCCL_MSCCL_ENABLE=0 \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-grok1-int4 --nightly --timeout-per-file 3600 || TEST_EXIT_CODE=$? + echo "$(> $GITHUB_STEP_SUMMARY || true + exit ${TEST_EXIT_CODE:-0} + + - name: Performance Test (8-GPU Grok1-INT4) + timeout-minutes: 60 + continue-on-error: true # Perf test failure doesn't fail the job if accuracy passed + run: | + > github_summary.md # Clear summary file + bash scripts/ci/amd_ci_exec.sh -w /sglang-checkout/test \ + -e RCCL_MSCCL_ENABLE=0 \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-grok1-int4 --nightly --timeout-per-file 3600 || TEST_EXIT_CODE=$? + echo "$(> $GITHUB_STEP_SUMMARY || true + exit ${TEST_EXIT_CODE:-0} + + # 8-GPU Grok2 (Accuracy + Performance combined) + nightly-8-gpu-grok2: + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-8-gpu-grok2') + runs-on: linux-mi325-gpu-8 + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref || github.ref }} + + - name: Setup docker + run: | + touch github_summary.md + bash scripts/ci/amd_ci_start_container.sh + env: + GITHUB_WORKSPACE: ${{ github.workspace }} + + - name: Install dependencies + run: bash scripts/ci/amd_ci_install_dependency.sh + + - name: Accuracy Test (8-GPU Grok2) + timeout-minutes: 60 + run: | + > github_summary.md # Clear summary file + bash scripts/ci/amd_ci_exec.sh -w /sglang-checkout/test \ + -e RCCL_MSCCL_ENABLE=0 \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-grok2 --nightly --timeout-per-file 3600 || TEST_EXIT_CODE=$? + echo "$(> $GITHUB_STEP_SUMMARY || true + exit ${TEST_EXIT_CODE:-0} + + - name: Performance Test (8-GPU Grok2) + timeout-minutes: 60 + continue-on-error: true # Perf test failure doesn't fail the job if accuracy passed + run: | + > github_summary.md # Clear summary file + bash scripts/ci/amd_ci_exec.sh -w /sglang-checkout/test \ + -e RCCL_MSCCL_ENABLE=0 \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-grok2 --nightly --timeout-per-file 3600 || TEST_EXIT_CODE=$? + echo "$(> $GITHUB_STEP_SUMMARY || true + exit ${TEST_EXIT_CODE:-0} + + # 8-GPU DeepSeek-V3.1 (Accuracy + Performance combined) + nightly-8-gpu-deepseek-v31: + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-8-gpu-deepseek-v31') + runs-on: linux-mi325-gpu-8 + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref || github.ref }} + + - name: Setup docker + run: | + touch github_summary.md + bash scripts/ci/amd_ci_start_container.sh + env: + GITHUB_WORKSPACE: ${{ github.workspace }} + + - name: Install dependencies + run: bash scripts/ci/amd_ci_install_dependency.sh + + - name: Accuracy Test (8-GPU DeepSeek-V3.1) + timeout-minutes: 120 + run: | + > github_summary.md # Clear summary file + bash scripts/ci/amd_ci_exec.sh -w /sglang-checkout/test \ + -e SGLANG_USE_AITER=1 \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-deepseek-v31 --nightly --timeout-per-file 3600 || TEST_EXIT_CODE=$? + echo "$(> $GITHUB_STEP_SUMMARY || true + exit ${TEST_EXIT_CODE:-0} + + - name: Performance Test (8-GPU DeepSeek-V3.1) + timeout-minutes: 300 + continue-on-error: true # Perf test failure doesn't fail the job if accuracy passed + run: | + > github_summary.md # Clear summary file + bash scripts/ci/amd_ci_exec.sh -w /sglang-checkout/test \ + -e SGLANG_USE_ROCM700A=1 \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-deepseek-v31 --nightly --timeout-per-file 18000 || TEST_EXIT_CODE=$? echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} # ============================================== MI35x Tests ============================================== - # MI35x 2-GPU tests (TP=2) - Reuses nightly-amd suite - nightly-test-2-gpu-mi35x: - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-2-gpu-mi35x') - runs-on: linux-mi35x-gpu-2 + # MI35x 1-GPU tests - platform-agnostic tests that may work on CDNA4 (gfx950) + nightly-test-1-gpu-mi35x: + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-1-gpu-mi35x') + runs-on: linux-mi35x-gpu-1 steps: - name: Checkout code uses: actions/checkout@v4 @@ -300,179 +363,18 @@ jobs: # Install tabulate for run_suite.py (missing in MI35x container) bash scripts/ci/amd_ci_exec.sh pip install tabulate - - name: Nightly Test (2-GPU) - run: | - bash scripts/ci/amd_ci_exec.sh -w /sglang-checkout/test \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-amd --nightly --timeout-per-file 7200 || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - # MI35x 2-GPU VLM tests - Reuses nightly-amd-vlm suite - nightly-test-2-gpu-vlm-mi35x: - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-2-gpu-vlm-mi35x') - runs-on: linux-mi35x-gpu-2 - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref || github.ref }} - - - name: Setup docker - run: | - touch github_summary.md - bash scripts/ci/amd_ci_start_container.sh - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - - - name: Install dependencies - run: | - bash scripts/ci/amd_ci_install_dependency.sh - # Install tabulate for run_suite.py (missing in MI35x container) - bash scripts/ci/amd_ci_exec.sh pip install tabulate - - - name: Nightly Test (2-GPU VLM MMMU) - timeout-minutes: 180 - run: | - bash scripts/ci/amd_ci_exec.sh -w /sglang-checkout/test \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-amd-vlm --nightly --timeout-per-file 7200 || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - # MI35x 8-GPU tests (TP=8) - GPT-OSS models (MI35x uses openai/* paths) - nightly-test-8-gpu-mi35x-gpt-oss: - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-8-gpu-mi35x-gpt-oss') - runs-on: linux-mi35x-gpu-8 - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref || github.ref }} - - - name: Setup docker - run: | - touch github_summary.md - bash scripts/ci/amd_ci_start_container.sh - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - - - name: Install dependencies - run: | - bash scripts/ci/amd_ci_install_dependency.sh - # Install tabulate for run_suite.py (missing in MI35x container) - bash scripts/ci/amd_ci_exec.sh pip install tabulate - - - name: Nightly Test MI35x (8-GPU GPT-OSS) - timeout-minutes: 180 - run: | - bash scripts/ci/amd_ci_exec.sh -w /sglang-checkout/test \ - -e AMD_TEST_MODEL_GROUP=gpt-oss \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x --nightly --timeout-per-file 7200 || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - # MI35x 8-GPU tests (TP=8) - GROK models - nightly-test-8-gpu-mi35x-grok: - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-8-gpu-mi35x-grok') - runs-on: linux-mi35x-gpu-8 - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref || github.ref }} - - - name: Setup docker - run: | - touch github_summary.md - bash scripts/ci/amd_ci_start_container.sh - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - - - name: Install dependencies - run: | - bash scripts/ci/amd_ci_install_dependency.sh - # Install tabulate for run_suite.py (missing in MI35x container) - bash scripts/ci/amd_ci_exec.sh pip install tabulate - - - name: Nightly Test MI35x (8-GPU GROK) - run: | - bash scripts/ci/amd_ci_exec.sh -w /sglang-checkout/test \ - -e AMD_TEST_MODEL_GROUP=grok \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-amd-8-gpu --nightly --timeout-per-file 7200 || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - # MI35x 8-GPU tests (TP=8) - DeepSeek-R1-0528 basic + MTP only - # Same model as MI300X for consistency; MXFP4 only used for perf tests - # Note: DP/TC variants disabled for MI35x due to initialization timeouts - nightly-test-8-gpu-mi35x-deepseek-r1: - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-8-gpu-mi35x-deepseek-r1') - runs-on: linux-mi35x-gpu-8 - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref || github.ref }} - - - name: Setup docker - run: | - touch github_summary.md - bash scripts/ci/amd_ci_start_container.sh - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - - - name: Install dependencies - run: | - bash scripts/ci/amd_ci_install_dependency.sh - # Install tabulate for run_suite.py (missing in MI35x container) - bash scripts/ci/amd_ci_exec.sh pip install tabulate - - - name: Nightly Test MI35x (8-GPU DeepSeek-R1-0528 basic + MTP) - timeout-minutes: 180 - run: | - bash scripts/ci/amd_ci_exec.sh -w /sglang-checkout/test \ - -e AMD_TEST_MODEL_GROUP=deepseek-r1 \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x --nightly --timeout-per-file 7200 || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - # MI35x 8-GPU Performance Tests (TP=8) - Grok performance benchmarks - nightly-perf-8-gpu-mi35x-grok: - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-perf-8-gpu-mi35x-grok') - runs-on: linux-mi35x-gpu-8 - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref || github.ref }} - - - name: Setup docker - run: | - touch github_summary.md - bash scripts/ci/amd_ci_start_container.sh - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - - - name: Install dependencies - run: | - bash scripts/ci/amd_ci_install_dependency.sh - # Install tabulate for run_suite.py (missing in MI35x container) - bash scripts/ci/amd_ci_exec.sh pip install tabulate - - - name: Nightly Perf Test MI35x (8-GPU Grok) + - name: Nightly Test MI35x (1-GPU) timeout-minutes: 60 run: | - bash scripts/ci/amd_ci_exec.sh -w /sglang-checkout/test -e RCCL_MSCCL_ENABLE=0 -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" python3 registered/amd/test_grok_perf.py || TEST_EXIT_CODE=$? + bash scripts/ci/amd_ci_exec.sh -w /sglang-checkout/test \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + python3 run_suite.py --hw amd --suite nightly-amd-1-gpu-mi35x --nightly --timeout-per-file 600 --continue-on-error || TEST_EXIT_CODE=$? echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} - # MI35x 8-GPU Performance Tests (TP=8) - DeepSeek-R1-MXFP4 performance benchmarks - nightly-perf-8-gpu-mi35x-deepseek-r1-mxfp4: - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-perf-8-gpu-mi35x-deepseek-r1-mxfp4') + # MI35x 8-GPU Accuracy Tests - GPT-OSS (accuracy only) + nightly-accuracy-8-gpu-mi35x: + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-accuracy-8-gpu-mi35x') runs-on: linux-mi35x-gpu-8 steps: - name: Checkout code @@ -493,33 +395,204 @@ jobs: # Install tabulate for run_suite.py (missing in MI35x container) bash scripts/ci/amd_ci_exec.sh pip install tabulate - - name: Nightly Perf Test MI35x (8-GPU DeepSeek-R1-MXFP4) - timeout-minutes: 300 + - name: Accuracy Test MI35x (8-GPU GPT-OSS) + timeout-minutes: 180 run: | - bash scripts/ci/amd_ci_exec.sh -w /sglang-checkout/test -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" python3 registered/amd/test_deepseek_r1_mxfp4_perf.py || TEST_EXIT_CODE=$? + bash scripts/ci/amd_ci_exec.sh -w /sglang-checkout/test \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x --nightly --timeout-per-file 7200 || TEST_EXIT_CODE=$? + echo "$(> $GITHUB_STEP_SUMMARY || true + exit ${TEST_EXIT_CODE:-0} + + # MI35x 8-GPU DeepSeek-R1-0528 Accuracy Test (separate job due to long loading time) + nightly-accuracy-8-gpu-mi35x-deepseek-r1: + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-accuracy-8-gpu-mi35x-deepseek-r1') + runs-on: linux-mi35x-gpu-8 + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref || github.ref }} + + - name: Setup docker + run: | + touch github_summary.md + bash scripts/ci/amd_ci_start_container.sh + env: + GITHUB_WORKSPACE: ${{ github.workspace }} + + - name: Install dependencies + run: | + bash scripts/ci/amd_ci_install_dependency.sh + # Install tabulate for run_suite.py (missing in MI35x container) + bash scripts/ci/amd_ci_exec.sh pip install tabulate + + - name: Accuracy Test MI35x (8-GPU DeepSeek-R1-0528) + timeout-minutes: 240 + run: | + bash scripts/ci/amd_ci_exec.sh -w /sglang-checkout/test \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-mi35x-deepseek-r1 --nightly --timeout-per-file 7200 || TEST_EXIT_CODE=$? + echo "$(> $GITHUB_STEP_SUMMARY || true + exit ${TEST_EXIT_CODE:-0} + + # MI35x 8-GPU Grok1-INT4 (Accuracy + Performance combined) + nightly-8-gpu-mi35x-grok1-int4: + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-8-gpu-mi35x-grok1-int4') + runs-on: linux-mi35x-gpu-8 + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref || github.ref }} + + - name: Setup docker + run: | + touch github_summary.md + bash scripts/ci/amd_ci_start_container.sh + env: + GITHUB_WORKSPACE: ${{ github.workspace }} + + - name: Install dependencies + run: | + bash scripts/ci/amd_ci_install_dependency.sh + # Install tabulate for run_suite.py (missing in MI35x container) + bash scripts/ci/amd_ci_exec.sh pip install tabulate + + - name: Accuracy Test MI35x (8-GPU Grok1-INT4) + timeout-minutes: 60 + run: | + > github_summary.md # Clear summary file + bash scripts/ci/amd_ci_exec.sh -w /sglang-checkout/test \ + -e RCCL_MSCCL_ENABLE=0 \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-mi35x-grok1-int4 --nightly --timeout-per-file 3600 || TEST_EXIT_CODE=$? + echo "$(> $GITHUB_STEP_SUMMARY || true + exit ${TEST_EXIT_CODE:-0} + + - name: Performance Test MI35x (8-GPU Grok1-INT4) + timeout-minutes: 60 + continue-on-error: true # Perf test failure doesn't fail the job if accuracy passed + run: | + > github_summary.md # Clear summary file + bash scripts/ci/amd_ci_exec.sh -w /sglang-checkout/test \ + -e RCCL_MSCCL_ENABLE=0 \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-mi35x-grok1-int4 --nightly --timeout-per-file 3600 || TEST_EXIT_CODE=$? + echo "$(> $GITHUB_STEP_SUMMARY || true + exit ${TEST_EXIT_CODE:-0} + + # MI35x 8-GPU Grok2 (Accuracy + Performance combined) + nightly-8-gpu-mi35x-grok2: + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-8-gpu-mi35x-grok2') + runs-on: linux-mi35x-gpu-8 + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref || github.ref }} + + - name: Setup docker + run: | + touch github_summary.md + bash scripts/ci/amd_ci_start_container.sh + env: + GITHUB_WORKSPACE: ${{ github.workspace }} + + - name: Install dependencies + run: | + bash scripts/ci/amd_ci_install_dependency.sh + # Install tabulate for run_suite.py (missing in MI35x container) + bash scripts/ci/amd_ci_exec.sh pip install tabulate + + - name: Accuracy Test MI35x (8-GPU Grok2) + timeout-minutes: 60 + run: | + > github_summary.md # Clear summary file + bash scripts/ci/amd_ci_exec.sh -w /sglang-checkout/test \ + -e RCCL_MSCCL_ENABLE=0 \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-mi35x-grok2 --nightly --timeout-per-file 3600 || TEST_EXIT_CODE=$? + echo "$(> $GITHUB_STEP_SUMMARY || true + exit ${TEST_EXIT_CODE:-0} + + - name: Performance Test MI35x (8-GPU Grok2) + timeout-minutes: 60 + continue-on-error: true # Perf test failure doesn't fail the job if accuracy passed + run: | + > github_summary.md # Clear summary file + bash scripts/ci/amd_ci_exec.sh -w /sglang-checkout/test \ + -e RCCL_MSCCL_ENABLE=0 \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-mi35x-grok2 --nightly --timeout-per-file 3600 || TEST_EXIT_CODE=$? + echo "$(> $GITHUB_STEP_SUMMARY || true + exit ${TEST_EXIT_CODE:-0} + + # MI35x 8-GPU DeepSeek-R1-MXFP4 (Accuracy + Performance combined) + nightly-8-gpu-mi35x-deepseek-r1-mxfp4: + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-8-gpu-mi35x-deepseek-r1-mxfp4') + runs-on: linux-mi35x-gpu-8 + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref || github.ref }} + + - name: Setup docker + run: | + touch github_summary.md + bash scripts/ci/amd_ci_start_container.sh + env: + GITHUB_WORKSPACE: ${{ github.workspace }} + + - name: Install dependencies + run: | + bash scripts/ci/amd_ci_install_dependency.sh + # Install tabulate for run_suite.py (missing in MI35x container) + bash scripts/ci/amd_ci_exec.sh pip install tabulate + + - name: Accuracy Test MI35x (8-GPU DeepSeek-R1-MXFP4) + timeout-minutes: 180 + run: | + > github_summary.md # Clear summary file + bash scripts/ci/amd_ci_exec.sh -w /sglang-checkout/test \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x-deepseek-r1-mxfp4 --nightly --timeout-per-file 7200 || TEST_EXIT_CODE=$? + echo "$(> $GITHUB_STEP_SUMMARY || true + exit ${TEST_EXIT_CODE:-0} + + - name: Performance Test MI35x (8-GPU DeepSeek-R1-MXFP4) + timeout-minutes: 300 + continue-on-error: true # Perf test failure doesn't fail the job if accuracy passed + run: | + > github_summary.md # Clear summary file + bash scripts/ci/amd_ci_exec.sh -w /sglang-checkout/test \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + python3 registered/amd/perf/mi35x/test_deepseek_r1_mxfp4_perf_mi35x.py || TEST_EXIT_CODE=$? echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} check-all-jobs: if: always() && (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch') needs: - # MI325 jobs - - nightly-test-2-gpu - - nightly-test-2-gpu-vlm - - nightly-test-8-gpu-gpt-oss - - nightly-test-8-gpu-grok - - nightly-test-8-gpu-deepseek-r1 - - nightly-perf-8-gpu-grok - - nightly-perf-8-gpu-deepseek-v3 - - nightly-perf-8-gpu-deepseek-v31 + # MI30x Unit Tests + - nightly-test-1-gpu-unit + # MI30x Accuracy Tests + - nightly-accuracy-2-gpu + - nightly-accuracy-2-gpu-vlm + - nightly-accuracy-8-gpu + - nightly-accuracy-8-gpu-deepseek-r1 + # MI30x Combined Accuracy + Performance Tests + - nightly-8-gpu-grok1-int4 + - nightly-8-gpu-grok2 + - nightly-8-gpu-deepseek-v31 # MI35x jobs - - nightly-test-2-gpu-mi35x - - nightly-test-2-gpu-vlm-mi35x - - nightly-test-8-gpu-mi35x-gpt-oss - - nightly-test-8-gpu-mi35x-grok - - nightly-test-8-gpu-mi35x-deepseek-r1 - - nightly-perf-8-gpu-mi35x-grok - - nightly-perf-8-gpu-mi35x-deepseek-r1-mxfp4 + - nightly-test-1-gpu-mi35x + - nightly-accuracy-8-gpu-mi35x + - nightly-accuracy-8-gpu-mi35x-deepseek-r1 + - nightly-8-gpu-mi35x-grok1-int4 + - nightly-8-gpu-mi35x-grok2 + - nightly-8-gpu-mi35x-deepseek-r1-mxfp4 runs-on: ubuntu-latest steps: - name: Check if any job failed diff --git a/test/registered/amd/accuracy/mi35x/test_deepseek_r1_eval_mi35x.py b/test/registered/amd/accuracy/mi35x/test_deepseek_r1_eval_mi35x.py new file mode 100644 index 000000000..d4dcc2838 --- /dev/null +++ b/test/registered/amd/accuracy/mi35x/test_deepseek_r1_eval_mi35x.py @@ -0,0 +1,250 @@ +"""MI35x DeepSeek-R1 GSM8K Completion Evaluation Test (8-GPU) + +Tests DeepSeek-R1-0528 with basic configuration using few-shot completion +benchmark on MI35x. + +Registry: nightly-amd-accuracy-8-gpu-mi35x-deepseek-r1 suite +""" + +import ast +import os + +# Set HF cache for MI35x +os.environ.setdefault("HF_HOME", "/data2/models/huggingface") +os.environ.setdefault("HF_HUB_CACHE", "/data2/models/huggingface/hub") + +import re +import time +import unittest +from dataclasses import dataclass +from typing import List, Optional, Tuple + +import numpy as np + +from sglang.srt.utils import kill_process_tree +from sglang.test.ci.ci_register import register_amd_ci +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + is_in_ci, + popen_launch_server, + write_github_step_summary, +) +from sglang.utils import download_and_cache_file, read_jsonl + +# Register for AMD CI - MI35x DeepSeek-R1-0528 accuracy test (~30 min for basic only) +register_amd_ci( + est_time=1800, + suite="nightly-amd-accuracy-8-gpu-mi35x-deepseek-r1", + nightly=True, +) + +INVALID = -9999999 + + +@dataclass +class ModelConfig: + """Configuration for a model to test.""" + + model_path: str + tp_size: int = 8 + accuracy_threshold: float = 0.50 + other_args: Optional[List[str]] = None + env_vars: Optional[dict] = None + timeout: Optional[int] = None + variant: Optional[str] = None + + def __post_init__(self): + if self.other_args is None: + self.other_args = [] + if self.env_vars is None: + self.env_vars = {} + + def get_display_name(self) -> str: + if self.variant: + return f"{self.model_path} ({self.variant})" + return self.model_path + + +# DeepSeek-R1 models for MI35x - only basic variant for faster CI +# MTP, DP, and TC variants removed to reduce test time from ~2h to ~30min +MI35X_DEEPSEEK_R1_MODELS = [ + # DeepSeek-R1-0528 basic + ModelConfig( + model_path="deepseek-ai/DeepSeek-R1-0528", + tp_size=8, + accuracy_threshold=0.93, + timeout=3600, + variant="basic", + other_args=[ + "--attention-backend", + "aiter", + "--chunked-prefill-size", + "131072", + "--disable-radix-cache", + "--mem-fraction-static", + "0.85", + "--trust-remote-code", + "--watchdog-timeout", + "1200", # 20 minutes for weight loading + ], + env_vars={"SGLANG_USE_AITER": "1"}, + ), +] + + +def get_one_example(lines, i, include_answer): + """Format a single GSM8K example.""" + ret = "Question: " + lines[i]["question"] + "\nAnswer:" + if include_answer: + ret += " " + lines[i]["answer"] + return ret + + +def get_few_shot_examples(lines, k): + """Get k few-shot examples for prompting.""" + ret = "" + for i in range(k): + ret += get_one_example(lines, i, True) + "\n\n" + return ret + + +def get_answer_value(answer_str): + """Extract numerical answer from response.""" + answer_str = answer_str.replace(",", "") + numbers = re.findall(r"\d+", answer_str) + if len(numbers) < 1: + return INVALID + try: + return ast.literal_eval(numbers[-1]) + except SyntaxError: + return INVALID + + +def run_gsm8k_benchmark( + base_url: str, + num_questions: int = 200, + num_shots: int = 5, + parallel: int = 64, +) -> Tuple[float, float, float]: + """Run GSM8K few-shot completion benchmark.""" + import sglang as sgl + from sglang.lang.backend.runtime_endpoint import RuntimeEndpoint + + url = "https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/test.jsonl" + data_path = download_and_cache_file(url) + lines = list(read_jsonl(data_path)) + + few_shot_examples = get_few_shot_examples(lines, num_shots) + + questions = [] + labels = [] + for i in range(len(lines[:num_questions])): + questions.append(get_one_example(lines, i, False)) + labels.append(get_answer_value(lines[i]["answer"])) + assert all(l != INVALID for l in labels) + arguments = [{"question": q} for q in questions] + + @sgl.function + def few_shot_gsm8k(s, question): + s += few_shot_examples + question + s += sgl.gen( + "answer", max_tokens=512, stop=["Question", "Assistant:", "<|separator|>"] + ) + + backend = RuntimeEndpoint(base_url) + sgl.set_default_backend(backend) + + tic = time.perf_counter() + states = few_shot_gsm8k.run_batch( + arguments, temperature=0, num_threads=parallel, progress_bar=True + ) + latency = time.perf_counter() - tic + + preds = [get_answer_value(states[i]["answer"]) for i in range(len(states))] + acc = np.mean(np.array(preds) == np.array(labels)) + invalid = np.mean(np.array(preds) == INVALID) + + return float(acc), float(invalid), float(latency) + + +class TestDeepSeekR1EvalMI35x(unittest.TestCase): + """DeepSeek-R1 GSM8K Completion Evaluation Test for AMD MI35x.""" + + @classmethod + def setUpClass(cls): + cls.models = MI35X_DEEPSEEK_R1_MODELS + cls.base_url = DEFAULT_URL_FOR_TEST + cls.num_questions = int(os.environ.get("GSM8K_NUM_QUESTIONS", "200")) + + def test_deepseek_r1_accuracy(self): + """Test DeepSeek-R1 models with GSM8K completion benchmark.""" + all_results = [] + summary = "### DeepSeek-R1 Models (MI35x)\n\n" + summary += "| Model | Variant | TP | Accuracy | Threshold | Status |\n" + summary += "| ----- | ------- | -- | -------- | --------- | ------ |\n" + + for config in self.models: + display_name = config.get_display_name() + with self.subTest(model=display_name): + print(f"\n{'='*60}") + print(f"Testing: {display_name}") + print(f"{'='*60}") + + env = os.environ.copy() + for key, value in config.env_vars.items(): + env[key] = value + + other_args = list(config.other_args) + other_args.extend(["--tp", str(config.tp_size)]) + timeout = config.timeout or DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH + + try: + process = popen_launch_server( + model=config.model_path, + base_url=self.base_url, + timeout=timeout, + other_args=other_args, + env=env, + ) + + try: + acc, invalid, latency = run_gsm8k_benchmark( + self.base_url, num_questions=self.num_questions + ) + passed = acc >= config.accuracy_threshold + status = "✅ PASS" if passed else "❌ FAIL" + + all_results.append( + { + "model": display_name, + "accuracy": acc, + "passed": passed, + } + ) + summary += f"| {config.model_path} | {config.variant or 'N/A'} | {config.tp_size} | {acc:.3f} | {config.accuracy_threshold} | {status} |\n" + + finally: + kill_process_tree(process.pid) + + except Exception as e: + summary += f"| {config.model_path} | {config.variant or 'N/A'} | {config.tp_size} | N/A | {config.accuracy_threshold} | ❌ ERROR |\n" + all_results.append( + { + "model": display_name, + "accuracy": None, + "passed": False, + "error": str(e), + } + ) + + if is_in_ci(): + write_github_step_summary(summary) + + failed = [r for r in all_results if not r["passed"]] + if failed: + raise AssertionError(f"Failed models: {[r['model'] for r in failed]}") + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/amd/accuracy/mi35x/test_deepseek_r1_mxfp4_eval_mi35x.py b/test/registered/amd/accuracy/mi35x/test_deepseek_r1_mxfp4_eval_mi35x.py new file mode 100644 index 000000000..e7e8f6b9a --- /dev/null +++ b/test/registered/amd/accuracy/mi35x/test_deepseek_r1_mxfp4_eval_mi35x.py @@ -0,0 +1,300 @@ +"""MI35x DeepSeek-R1-MXFP4 GSM8K Completion Evaluation Test (8-GPU) + +Tests DeepSeek-R1-MXFP4 quantized model with multiple configurations +(basic, MTP, DP, TC) using few-shot completion benchmark on MI35x. + +Registry: nightly-amd-8-gpu-mi35x-deepseek-r1-mxfp4 suite +""" + +import ast +import os + +# Set HF cache for MI35x +os.environ.setdefault("HF_HOME", "/data2/models/huggingface") +os.environ.setdefault("HF_HUB_CACHE", "/data2/models/huggingface/hub") + +import re +import time +import unittest +from dataclasses import dataclass +from typing import List, Optional, Tuple + +import numpy as np + +from sglang.srt.utils import kill_process_tree +from sglang.test.ci.ci_register import register_amd_ci +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + is_in_ci, + popen_launch_server, + write_github_step_summary, +) +from sglang.utils import download_and_cache_file, read_jsonl + +# Register for AMD CI - MI35x DeepSeek-R1-MXFP4 accuracy tests (~120 min) +register_amd_ci( + est_time=7200, suite="nightly-amd-8-gpu-mi35x-deepseek-r1-mxfp4", nightly=True +) + +INVALID = -9999999 + +# Model path configuration for MI35x DeepSeek-R1-MXFP4 +# Priority: 1) env var, 2) local path, 3) HuggingFace model ID +DEEPSEEK_R1_MXFP4_LOCAL_PATH = "/data2/models/amd-DeepSeek-R1-MXFP4-Preview" +DEEPSEEK_R1_MXFP4_HF_MODEL_ID = "amd/DeepSeek-R1-MXFP4-Preview" + + +def get_model_path() -> str: + """Get effective model path: env var > local path > HF model ID.""" + env_path = os.environ.get("DEEPSEEK_R1_MXFP4_MODEL_PATH") + if env_path: + return env_path + if os.path.exists(DEEPSEEK_R1_MXFP4_LOCAL_PATH): + return DEEPSEEK_R1_MXFP4_LOCAL_PATH + return DEEPSEEK_R1_MXFP4_HF_MODEL_ID + + +@dataclass +class ModelConfig: + """Configuration for a model to test.""" + + model_path: str + tp_size: int = 8 + accuracy_threshold: float = 0.50 + other_args: Optional[List[str]] = None + env_vars: Optional[dict] = None + timeout: Optional[int] = None + variant: Optional[str] = None + + def __post_init__(self): + if self.other_args is None: + self.other_args = [] + if self.env_vars is None: + self.env_vars = {} + + def get_display_name(self) -> str: + if self.variant: + return f"{self.model_path} ({self.variant})" + return self.model_path + + +def get_mxfp4_models() -> List[ModelConfig]: + """Get DeepSeek-R1-MXFP4 model configurations for MI35x.""" + model_path = get_model_path() + return [ + # DeepSeek-R1-MXFP4 basic + ModelConfig( + model_path=model_path, + tp_size=8, + accuracy_threshold=0.93, + timeout=3600, + variant="basic", + other_args=[ + "--attention-backend", + "aiter", + "--chunked-prefill-size", + "131072", + "--disable-radix-cache", + "--mem-fraction-static", + "0.85", + "--trust-remote-code", + ], + env_vars={"SGLANG_USE_AITER": "1"}, + ), + # DeepSeek-R1-MXFP4 with MTP (EAGLE) + ModelConfig( + model_path=model_path, + tp_size=8, + accuracy_threshold=0.93, + timeout=3600, + variant="MTP", + other_args=[ + "--chunked-prefill-size", + "131072", + "--speculative-algorithm", + "EAGLE", + "--speculative-num-steps", + "3", + "--speculative-eagle-topk", + "1", + "--speculative-num-draft-tokens", + "4", + "--mem-fraction-static", + "0.7", + "--trust-remote-code", + ], + env_vars={"SGLANG_USE_AITER": "1"}, + ), + # Note: DP and TC variants are not supported for MXFP4 on MI35x + ] + + +def get_one_example(lines, i, include_answer): + """Format a single GSM8K example.""" + ret = "Question: " + lines[i]["question"] + "\nAnswer:" + if include_answer: + ret += " " + lines[i]["answer"] + return ret + + +def get_few_shot_examples(lines, k): + """Get k few-shot examples for prompting.""" + ret = "" + for i in range(k): + ret += get_one_example(lines, i, True) + "\n\n" + return ret + + +def get_answer_value(answer_str): + """Extract numerical answer from response.""" + answer_str = answer_str.replace(",", "") + numbers = re.findall(r"\d+", answer_str) + if len(numbers) < 1: + return INVALID + try: + return ast.literal_eval(numbers[-1]) + except SyntaxError: + return INVALID + + +def run_gsm8k_benchmark( + base_url: str, + num_questions: int = 200, + num_shots: int = 5, + parallel: int = 64, +) -> Tuple[float, float, float]: + """Run GSM8K few-shot completion benchmark.""" + import sglang as sgl + from sglang.lang.backend.runtime_endpoint import RuntimeEndpoint + + url = "https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/test.jsonl" + data_path = download_and_cache_file(url) + lines = list(read_jsonl(data_path)) + + few_shot_examples = get_few_shot_examples(lines, num_shots) + + questions = [] + labels = [] + for i in range(len(lines[:num_questions])): + questions.append(get_one_example(lines, i, False)) + labels.append(get_answer_value(lines[i]["answer"])) + assert all(l != INVALID for l in labels) + arguments = [{"question": q} for q in questions] + + @sgl.function + def few_shot_gsm8k(s, question): + s += few_shot_examples + question + s += sgl.gen( + "answer", max_tokens=512, stop=["Question", "Assistant:", "<|separator|>"] + ) + + backend = RuntimeEndpoint(base_url) + sgl.set_default_backend(backend) + + tic = time.perf_counter() + states = few_shot_gsm8k.run_batch( + arguments, temperature=0, num_threads=parallel, progress_bar=True + ) + latency = time.perf_counter() - tic + + preds = [get_answer_value(states[i]["answer"]) for i in range(len(states))] + acc = np.mean(np.array(preds) == np.array(labels)) + invalid = np.mean(np.array(preds) == INVALID) + + return float(acc), float(invalid), float(latency) + + +class TestDeepSeekR1MXFP4EvalMI35x(unittest.TestCase): + """DeepSeek-R1-MXFP4 GSM8K Completion Evaluation Test for AMD MI35x.""" + + @classmethod + def setUpClass(cls): + cls.models = get_mxfp4_models() + cls.base_url = DEFAULT_URL_FOR_TEST + cls.num_questions = int(os.environ.get("GSM8K_NUM_QUESTIONS", "200")) + + def test_deepseek_r1_mxfp4_accuracy(self): + """Test DeepSeek-R1-MXFP4 models with GSM8K completion benchmark.""" + # Check if model exists + model_path = get_model_path() + is_local_path = model_path.startswith("/") + if is_local_path and not os.path.exists(model_path): + print(f"\n⏭️ SKIPPING: Local model not found at {model_path}") + self.skipTest(f"Local model not found at {model_path}") + return + + if is_local_path: + print(f"📁 Using local model: {model_path}") + else: + print(f"📥 Using HuggingFace model: {model_path}") + + all_results = [] + summary = "### DeepSeek-R1-MXFP4 Models (MI35x)\n\n" + summary += "| Model | Variant | TP | Accuracy | Threshold | Status |\n" + summary += "| ----- | ------- | -- | -------- | --------- | ------ |\n" + + for config in self.models: + display_name = config.get_display_name() + with self.subTest(model=display_name): + print(f"\n{'='*60}") + print(f"Testing: {display_name}") + print(f"{'='*60}") + + env = os.environ.copy() + for key, value in config.env_vars.items(): + env[key] = value + + other_args = list(config.other_args) + other_args.extend(["--tp", str(config.tp_size)]) + timeout = config.timeout or DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH + + try: + process = popen_launch_server( + model=config.model_path, + base_url=self.base_url, + timeout=timeout, + other_args=other_args, + env=env, + ) + + try: + acc, invalid, latency = run_gsm8k_benchmark( + self.base_url, num_questions=self.num_questions + ) + passed = acc >= config.accuracy_threshold + status = "✅ PASS" if passed else "❌ FAIL" + + all_results.append( + { + "model": display_name, + "accuracy": acc, + "passed": passed, + } + ) + summary += f"| {config.model_path} | {config.variant or 'N/A'} | {config.tp_size} | {acc:.3f} | {config.accuracy_threshold} | {status} |\n" + + finally: + kill_process_tree(process.pid) + + except Exception as e: + summary += f"| {config.model_path} | {config.variant or 'N/A'} | {config.tp_size} | N/A | {config.accuracy_threshold} | ❌ ERROR |\n" + all_results.append( + { + "model": display_name, + "accuracy": None, + "passed": False, + "error": str(e), + } + ) + + if is_in_ci(): + write_github_step_summary(summary) + + failed = [r for r in all_results if not r["passed"]] + if failed: + raise AssertionError(f"Failed models: {[r['model'] for r in failed]}") + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/amd/accuracy/mi35x/test_gpt_oss_eval_mi35x.py b/test/registered/amd/accuracy/mi35x/test_gpt_oss_eval_mi35x.py new file mode 100644 index 000000000..4e1451b4e --- /dev/null +++ b/test/registered/amd/accuracy/mi35x/test_gpt_oss_eval_mi35x.py @@ -0,0 +1,254 @@ +"""MI35x GPT-OSS GSM8K Completion Evaluation Test (8-GPU) + +Tests GPT-OSS models (openai/gpt-oss-20b, openai/gpt-oss-120b) using +few-shot completion benchmark on MI35x. + +Note: MI35x uses openai/* paths, not lmsys/* paths like MI300X. + +Registry: nightly-amd-8-gpu-mi35x suite +""" + +import ast +import os + +# Set HF cache for MI35x +os.environ.setdefault("HF_HOME", "/data2/models/huggingface") +os.environ.setdefault("HF_HUB_CACHE", "/data2/models/huggingface/hub") + +import re +import time +import unittest +from dataclasses import dataclass +from typing import List, Optional, Tuple + +import numpy as np + +from sglang.srt.utils import kill_process_tree +from sglang.test.ci.ci_register import register_amd_ci +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + is_in_ci, + popen_launch_server, + write_github_step_summary, +) +from sglang.utils import download_and_cache_file, read_jsonl + +# Register for AMD CI - MI35x GPT-OSS accuracy tests (~30 min) +register_amd_ci(est_time=1800, suite="nightly-amd-8-gpu-mi35x", nightly=True) + +INVALID = -9999999 + + +@dataclass +class ModelConfig: + """Configuration for a model to test.""" + + model_path: str + tp_size: int = 8 + accuracy_threshold: float = 0.50 + other_args: Optional[List[str]] = None + env_vars: Optional[dict] = None + timeout: Optional[int] = None + + def __post_init__(self): + if self.other_args is None: + self.other_args = [] + if self.env_vars is None: + self.env_vars = {} + + +# GPT-OSS models for MI35x (different paths from MI300X) +MI35X_GPT_OSS_MODELS = [ + ModelConfig( + model_path="openai/gpt-oss-20b", + tp_size=8, + accuracy_threshold=0.47, + other_args=[ + "--chunked-prefill-size", + "130172", + "--max-running-requests", + "128", + "--mem-fraction-static", + "0.85", + "--attention-backend", + "triton", + "--trust-remote-code", + ], + env_vars={"SGLANG_USE_AITER": "1"}, + ), + ModelConfig( + model_path="openai/gpt-oss-120b", + tp_size=8, + accuracy_threshold=0.79, + timeout=900, + other_args=[ + "--chunked-prefill-size", + "130172", + "--max-running-requests", + "128", + "--mem-fraction-static", + "0.85", + "--attention-backend", + "triton", + "--trust-remote-code", + ], + env_vars={"SGLANG_USE_AITER": "1"}, + ), +] + + +def get_one_example(lines, i, include_answer): + """Format a single GSM8K example.""" + ret = "Question: " + lines[i]["question"] + "\nAnswer:" + if include_answer: + ret += " " + lines[i]["answer"] + return ret + + +def get_few_shot_examples(lines, k): + """Get k few-shot examples for prompting.""" + ret = "" + for i in range(k): + ret += get_one_example(lines, i, True) + "\n\n" + return ret + + +def get_answer_value(answer_str): + """Extract numerical answer from response.""" + answer_str = answer_str.replace(",", "") + numbers = re.findall(r"\d+", answer_str) + if len(numbers) < 1: + return INVALID + try: + return ast.literal_eval(numbers[-1]) + except SyntaxError: + return INVALID + + +def run_gsm8k_benchmark( + base_url: str, + num_questions: int = 200, + num_shots: int = 5, + parallel: int = 64, +) -> Tuple[float, float, float]: + """Run GSM8K few-shot completion benchmark.""" + import sglang as sgl + from sglang.lang.backend.runtime_endpoint import RuntimeEndpoint + + url = "https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/test.jsonl" + data_path = download_and_cache_file(url) + lines = list(read_jsonl(data_path)) + + few_shot_examples = get_few_shot_examples(lines, num_shots) + + questions = [] + labels = [] + for i in range(len(lines[:num_questions])): + questions.append(get_one_example(lines, i, False)) + labels.append(get_answer_value(lines[i]["answer"])) + assert all(l != INVALID for l in labels) + arguments = [{"question": q} for q in questions] + + @sgl.function + def few_shot_gsm8k(s, question): + s += few_shot_examples + question + s += sgl.gen( + "answer", max_tokens=512, stop=["Question", "Assistant:", "<|separator|>"] + ) + + backend = RuntimeEndpoint(base_url) + sgl.set_default_backend(backend) + + tic = time.perf_counter() + states = few_shot_gsm8k.run_batch( + arguments, temperature=0, num_threads=parallel, progress_bar=True + ) + latency = time.perf_counter() - tic + + preds = [get_answer_value(states[i]["answer"]) for i in range(len(states))] + acc = np.mean(np.array(preds) == np.array(labels)) + invalid = np.mean(np.array(preds) == INVALID) + + return float(acc), float(invalid), float(latency) + + +class TestGptOssEvalMI35x(unittest.TestCase): + """GPT-OSS GSM8K Completion Evaluation Test for AMD MI35x.""" + + @classmethod + def setUpClass(cls): + cls.models = MI35X_GPT_OSS_MODELS + cls.base_url = DEFAULT_URL_FOR_TEST + cls.num_questions = int(os.environ.get("GSM8K_NUM_QUESTIONS", "200")) + + def test_gpt_oss_accuracy(self): + """Test GPT-OSS models with GSM8K completion benchmark.""" + all_results = [] + summary = "### GPT-OSS Models (MI35x)\n\n" + summary += "| Model | TP | Accuracy | Threshold | Status |\n" + summary += "| ----- | -- | -------- | --------- | ------ |\n" + + for config in self.models: + with self.subTest(model=config.model_path): + print(f"\n{'='*60}") + print(f"Testing: {config.model_path}") + print(f"{'='*60}") + + env = os.environ.copy() + for key, value in config.env_vars.items(): + env[key] = value + + other_args = list(config.other_args) + other_args.extend(["--tp", str(config.tp_size)]) + timeout = config.timeout or DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH + + try: + process = popen_launch_server( + model=config.model_path, + base_url=self.base_url, + timeout=timeout, + other_args=other_args, + env=env, + ) + + try: + acc, invalid, latency = run_gsm8k_benchmark( + self.base_url, num_questions=self.num_questions + ) + passed = acc >= config.accuracy_threshold + status = "✅ PASS" if passed else "❌ FAIL" + + all_results.append( + { + "model": config.model_path, + "accuracy": acc, + "passed": passed, + } + ) + summary += f"| {config.model_path} | {config.tp_size} | {acc:.3f} | {config.accuracy_threshold} | {status} |\n" + + finally: + kill_process_tree(process.pid) + + except Exception as e: + summary += f"| {config.model_path} | {config.tp_size} | N/A | {config.accuracy_threshold} | ❌ ERROR |\n" + all_results.append( + { + "model": config.model_path, + "accuracy": None, + "passed": False, + "error": str(e), + } + ) + + if is_in_ci(): + write_github_step_summary(summary) + + failed = [r for r in all_results if not r["passed"]] + if failed: + raise AssertionError(f"Failed models: {[r['model'] for r in failed]}") + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/amd/accuracy/mi35x/test_grok1_int4_eval_mi35x.py b/test/registered/amd/accuracy/mi35x/test_grok1_int4_eval_mi35x.py new file mode 100644 index 000000000..ecee813cc --- /dev/null +++ b/test/registered/amd/accuracy/mi35x/test_grok1_int4_eval_mi35x.py @@ -0,0 +1,162 @@ +"""MI35x GROK1-INT4 GSM8K Completion Evaluation Test (8-GPU) + +Tests Grok-1 INT4 (W4A8KV8) model using few-shot completion benchmark on MI35x. + +Registry: nightly-amd-accuracy-8-gpu-mi35x-grok1-int4 suite +""" + +import ast +import os +import re +import time +import unittest + +import numpy as np + +from sglang.srt.utils import kill_process_tree +from sglang.test.ci.ci_register import register_amd_ci +from sglang.test.test_utils import ( + DEFAULT_URL_FOR_TEST, + is_in_ci, + popen_launch_server, + write_github_step_summary, +) +from sglang.utils import download_and_cache_file, read_jsonl + +# Register for AMD CI - GROK1-INT4 accuracy tests on MI35x (~25 min) +register_amd_ci( + est_time=1500, suite="nightly-amd-accuracy-8-gpu-mi35x-grok1-int4", nightly=True +) + +INVALID = -9999999 + +GROK1_INT4_MODEL_PATH = os.environ.get("GROK1_INT4_MODEL_PATH", "amd/grok-1-W4A8KV8") +GROK1_TOKENIZER_PATH = os.environ.get("GROK1_TOKENIZER_PATH", "Xenova/grok-1-tokenizer") + + +def get_one_example(lines, i, include_answer): + ret = "Question: " + lines[i]["question"] + "\nAnswer:" + if include_answer: + ret += " " + lines[i]["answer"] + return ret + + +def get_few_shot_examples(lines, k): + ret = "" + for i in range(k): + ret += get_one_example(lines, i, True) + "\n\n" + return ret + + +def get_answer_value(answer_str): + answer_str = answer_str.replace(",", "") + numbers = re.findall(r"\d+", answer_str) + if len(numbers) < 1: + return INVALID + try: + return ast.literal_eval(numbers[-1]) + except SyntaxError: + return INVALID + + +def run_gsm8k_benchmark(base_url, num_questions=200, num_shots=5, parallel=64): + import sglang as sgl + from sglang.lang.backend.runtime_endpoint import RuntimeEndpoint + + url = "https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/test.jsonl" + data_path = download_and_cache_file(url) + lines = list(read_jsonl(data_path)) + + few_shot_examples = get_few_shot_examples(lines, num_shots) + questions = [] + labels = [] + for i in range(len(lines[:num_questions])): + questions.append(get_one_example(lines, i, False)) + labels.append(get_answer_value(lines[i]["answer"])) + arguments = [{"question": q} for q in questions] + + @sgl.function + def few_shot_gsm8k(s, question): + s += few_shot_examples + question + s += sgl.gen( + "answer", max_tokens=512, stop=["Question", "Assistant:", "<|separator|>"] + ) + + backend = RuntimeEndpoint(base_url) + sgl.set_default_backend(backend) + + tic = time.perf_counter() + states = few_shot_gsm8k.run_batch( + arguments, temperature=0, num_threads=parallel, progress_bar=True + ) + latency = time.perf_counter() - tic + + preds = [get_answer_value(states[i]["answer"]) for i in range(len(states))] + acc = np.mean(np.array(preds) == np.array(labels)) + return float(acc), float(latency) + + +class TestGrok1INT4EvalMI35x(unittest.TestCase): + """GROK1-INT4 GSM8K Completion Evaluation Test for MI35x.""" + + @classmethod + def setUpClass(cls): + cls.base_url = DEFAULT_URL_FOR_TEST + cls.num_questions = int(os.environ.get("GSM8K_NUM_QUESTIONS", "200")) + cls.accuracy_threshold = 0.80 + + def test_grok1_int4_accuracy(self): + """Test Grok-1 INT4 with GSM8K completion benchmark.""" + env = os.environ.copy() + env["RCCL_MSCCL_ENABLE"] = "0" + env["SGLANG_USE_AITER"] = "1" + env["SGLANG_INT4_WEIGHT"] = "1" + + other_args = [ + "--tp", + "8", + "--quantization", + "fp8", + "--attention-backend", + "aiter", + "--mem-fraction-static", + "0.85", + "--tokenizer-path", + GROK1_TOKENIZER_PATH, + "--trust-remote-code", + ] + + process = popen_launch_server( + model=GROK1_INT4_MODEL_PATH, + base_url=self.base_url, + timeout=3600, + other_args=other_args, + env=env, + ) + + try: + acc, latency = run_gsm8k_benchmark( + self.base_url, num_questions=self.num_questions + ) + passed = acc >= self.accuracy_threshold + status = "✅ PASS" if passed else "❌ FAIL" + + summary = f"### GROK1-INT4 (MI35x)\n\n" + summary += f"| Model | Accuracy | Threshold | Status |\n" + summary += f"| ----- | -------- | --------- | ------ |\n" + summary += f"| {GROK1_INT4_MODEL_PATH} | {acc:.3f} | {self.accuracy_threshold} | {status} |\n" + + if is_in_ci(): + write_github_step_summary(summary) + + self.assertGreaterEqual( + acc, + self.accuracy_threshold, + f"Accuracy {acc:.3f} below threshold {self.accuracy_threshold}", + ) + finally: + kill_process_tree(process.pid) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/amd/accuracy/mi35x/test_grok2_eval_mi35x.py b/test/registered/amd/accuracy/mi35x/test_grok2_eval_mi35x.py new file mode 100644 index 000000000..095ba3544 --- /dev/null +++ b/test/registered/amd/accuracy/mi35x/test_grok2_eval_mi35x.py @@ -0,0 +1,164 @@ +"""MI35x GROK2 GSM8K Completion Evaluation Test (8-GPU) + +Tests Grok-2 model using few-shot completion benchmark on MI35x. + +Registry: nightly-amd-accuracy-8-gpu-mi35x-grok2 suite +""" + +import ast +import os +import re +import time +import unittest + +import numpy as np + +from sglang.srt.utils import kill_process_tree +from sglang.test.ci.ci_register import register_amd_ci +from sglang.test.test_utils import ( + DEFAULT_URL_FOR_TEST, + is_in_ci, + popen_launch_server, + write_github_step_summary, +) +from sglang.utils import download_and_cache_file, read_jsonl + +# Register for AMD CI - GROK2 accuracy tests on MI35x (~25 min) +register_amd_ci( + est_time=1500, suite="nightly-amd-accuracy-8-gpu-mi35x-grok2", nightly=True +) + +INVALID = -9999999 + +GROK2_MODEL_PATH = os.environ.get("GROK2_MODEL_PATH", "xai-org/grok-2") +GROK2_TOKENIZER_PATH = os.environ.get( + "GROK2_TOKENIZER_PATH", "alvarobartt/grok-2-tokenizer" +) + + +def get_one_example(lines, i, include_answer): + ret = "Question: " + lines[i]["question"] + "\nAnswer:" + if include_answer: + ret += " " + lines[i]["answer"] + return ret + + +def get_few_shot_examples(lines, k): + ret = "" + for i in range(k): + ret += get_one_example(lines, i, True) + "\n\n" + return ret + + +def get_answer_value(answer_str): + answer_str = answer_str.replace(",", "") + numbers = re.findall(r"\d+", answer_str) + if len(numbers) < 1: + return INVALID + try: + return ast.literal_eval(numbers[-1]) + except SyntaxError: + return INVALID + + +def run_gsm8k_benchmark(base_url, num_questions=200, num_shots=5, parallel=64): + import sglang as sgl + from sglang.lang.backend.runtime_endpoint import RuntimeEndpoint + + url = "https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/test.jsonl" + data_path = download_and_cache_file(url) + lines = list(read_jsonl(data_path)) + + few_shot_examples = get_few_shot_examples(lines, num_shots) + questions = [] + labels = [] + for i in range(len(lines[:num_questions])): + questions.append(get_one_example(lines, i, False)) + labels.append(get_answer_value(lines[i]["answer"])) + arguments = [{"question": q} for q in questions] + + @sgl.function + def few_shot_gsm8k(s, question): + s += few_shot_examples + question + s += sgl.gen( + "answer", max_tokens=512, stop=["Question", "Assistant:", "<|separator|>"] + ) + + backend = RuntimeEndpoint(base_url) + sgl.set_default_backend(backend) + + tic = time.perf_counter() + states = few_shot_gsm8k.run_batch( + arguments, temperature=0, num_threads=parallel, progress_bar=True + ) + latency = time.perf_counter() - tic + + preds = [get_answer_value(states[i]["answer"]) for i in range(len(states))] + acc = np.mean(np.array(preds) == np.array(labels)) + return float(acc), float(latency) + + +class TestGrok2EvalMI35x(unittest.TestCase): + """GROK2 GSM8K Completion Evaluation Test for MI35x.""" + + @classmethod + def setUpClass(cls): + cls.base_url = DEFAULT_URL_FOR_TEST + cls.num_questions = int(os.environ.get("GSM8K_NUM_QUESTIONS", "200")) + cls.accuracy_threshold = 0.915 + + def test_grok2_accuracy(self): + """Test Grok-2 with GSM8K completion benchmark.""" + env = os.environ.copy() + env["RCCL_MSCCL_ENABLE"] = "0" + env["SGLANG_USE_AITER"] = "1" + env["SGLANG_INT4_WEIGHT"] = "0" + + other_args = [ + "--tp", + "8", + "--quantization", + "fp8", + "--attention-backend", + "aiter", + "--mem-fraction-static", + "0.85", + "--tokenizer-path", + GROK2_TOKENIZER_PATH, + "--trust-remote-code", + ] + + process = popen_launch_server( + model=GROK2_MODEL_PATH, + base_url=self.base_url, + timeout=3600, + other_args=other_args, + env=env, + ) + + try: + acc, latency = run_gsm8k_benchmark( + self.base_url, num_questions=self.num_questions + ) + passed = acc >= self.accuracy_threshold + status = "✅ PASS" if passed else "❌ FAIL" + + summary = f"### GROK2 (MI35x)\n\n" + summary += f"| Model | Accuracy | Threshold | Status |\n" + summary += f"| ----- | -------- | --------- | ------ |\n" + summary += f"| {GROK2_MODEL_PATH} | {acc:.3f} | {self.accuracy_threshold} | {status} |\n" + + if is_in_ci(): + write_github_step_summary(summary) + + self.assertGreaterEqual( + acc, + self.accuracy_threshold, + f"Accuracy {acc:.3f} below threshold {self.accuracy_threshold}", + ) + finally: + kill_process_tree(process.pid) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/amd/accuracy/test_deepseek_r1_eval_amd.py b/test/registered/amd/accuracy/test_deepseek_r1_eval_amd.py new file mode 100644 index 000000000..89fff1f74 --- /dev/null +++ b/test/registered/amd/accuracy/test_deepseek_r1_eval_amd.py @@ -0,0 +1,309 @@ +"""AMD DeepSeek-R1 GSM8K Completion Evaluation Test (8-GPU) + +Tests DeepSeek-R1-0528 with multiple configurations (basic, MTP, DP, TC) +using few-shot completion benchmark on MI300X. + +Registry: nightly-amd-8-gpu-deepseek-r1 suite +""" + +import ast +import os +import re +import time +import unittest +from dataclasses import dataclass +from typing import List, Optional, Tuple + +import numpy as np + +from sglang.srt.utils import kill_process_tree +from sglang.test.ci.ci_register import register_amd_ci +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + is_in_ci, + popen_launch_server, + write_github_step_summary, +) +from sglang.utils import download_and_cache_file, read_jsonl + +# Register for AMD CI - DeepSeek-R1 accuracy tests (~120 min) +register_amd_ci( + est_time=7200, suite="nightly-amd-accuracy-8-gpu-deepseek-r1", nightly=True +) + +INVALID = -9999999 + + +@dataclass +class ModelConfig: + """Configuration for a model to test.""" + + model_path: str + tp_size: int = 8 + accuracy_threshold: float = 0.50 + other_args: Optional[List[str]] = None + env_vars: Optional[dict] = None + timeout: Optional[int] = None + variant: Optional[str] = None + + def __post_init__(self): + if self.other_args is None: + self.other_args = [] + if self.env_vars is None: + self.env_vars = {} + + def get_display_name(self) -> str: + if self.variant: + return f"{self.model_path} ({self.variant})" + return self.model_path + + +# DeepSeek-R1 models for MI300X +DEEPSEEK_R1_MODELS = [ + # DeepSeek-R1-0528 basic + ModelConfig( + model_path="deepseek-ai/DeepSeek-R1-0528", + tp_size=8, + accuracy_threshold=0.93, + timeout=3600, + variant="basic", + other_args=[ + "--attention-backend", + "aiter", + "--chunked-prefill-size", + "131072", + "--disable-radix-cache", + "--mem-fraction-static", + "0.85", + "--trust-remote-code", + ], + env_vars={"SGLANG_USE_AITER": "1"}, + ), + # DeepSeek-R1-0528 with MTP (EAGLE) + ModelConfig( + model_path="deepseek-ai/DeepSeek-R1-0528", + tp_size=8, + accuracy_threshold=0.93, + timeout=3600, + variant="MTP", + other_args=[ + "--chunked-prefill-size", + "131072", + "--speculative-algorithm", + "EAGLE", + "--speculative-num-steps", + "3", + "--speculative-eagle-topk", + "1", + "--speculative-num-draft-tokens", + "4", + "--mem-fraction-static", + "0.7", + "--trust-remote-code", + ], + env_vars={"SGLANG_USE_AITER": "1"}, + ), + # DeepSeek-R1-0528 with DP attention + ModelConfig( + model_path="deepseek-ai/DeepSeek-R1-0528", + tp_size=8, + accuracy_threshold=0.93, + timeout=3600, + variant="DP", + other_args=[ + "--chunked-prefill-size", + "131072", + "--dp-size", + "8", + "--enable-dp-attention", + "--mem-fraction-static", + "0.85", + "--trust-remote-code", + ], + env_vars={ + "SGLANG_USE_ROCM700A": "1", + "SGLANG_USE_AITER": "1", + }, + ), + # DeepSeek-R1-0528 with torch compile + ModelConfig( + model_path="deepseek-ai/DeepSeek-R1-0528", + tp_size=8, + accuracy_threshold=0.93, + timeout=7200, + variant="TC", + other_args=[ + "--chunked-prefill-size", + "131072", + "--mem-fraction-static", + "0.70", + "--cuda-graph-max-bs", + "8", + "--enable-torch-compile", + "--disable-cuda-graph", + "--trust-remote-code", + ], + env_vars={ + "SGLANG_USE_ROCM700A": "1", + "SGLANG_USE_AITER": "1", + }, + ), +] + + +def get_one_example(lines, i, include_answer): + """Format a single GSM8K example.""" + ret = "Question: " + lines[i]["question"] + "\nAnswer:" + if include_answer: + ret += " " + lines[i]["answer"] + return ret + + +def get_few_shot_examples(lines, k): + """Get k few-shot examples for prompting.""" + ret = "" + for i in range(k): + ret += get_one_example(lines, i, True) + "\n\n" + return ret + + +def get_answer_value(answer_str): + """Extract numerical answer from response.""" + answer_str = answer_str.replace(",", "") + numbers = re.findall(r"\d+", answer_str) + if len(numbers) < 1: + return INVALID + try: + return ast.literal_eval(numbers[-1]) + except SyntaxError: + return INVALID + + +def run_gsm8k_benchmark( + base_url: str, + num_questions: int = 200, + num_shots: int = 5, + parallel: int = 64, +) -> Tuple[float, float, float]: + """Run GSM8K few-shot completion benchmark.""" + import sglang as sgl + from sglang.lang.backend.runtime_endpoint import RuntimeEndpoint + + url = "https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/test.jsonl" + data_path = download_and_cache_file(url) + lines = list(read_jsonl(data_path)) + + few_shot_examples = get_few_shot_examples(lines, num_shots) + + questions = [] + labels = [] + for i in range(len(lines[:num_questions])): + questions.append(get_one_example(lines, i, False)) + labels.append(get_answer_value(lines[i]["answer"])) + assert all(l != INVALID for l in labels) + arguments = [{"question": q} for q in questions] + + @sgl.function + def few_shot_gsm8k(s, question): + s += few_shot_examples + question + s += sgl.gen( + "answer", max_tokens=512, stop=["Question", "Assistant:", "<|separator|>"] + ) + + backend = RuntimeEndpoint(base_url) + sgl.set_default_backend(backend) + + tic = time.perf_counter() + states = few_shot_gsm8k.run_batch( + arguments, temperature=0, num_threads=parallel, progress_bar=True + ) + latency = time.perf_counter() - tic + + preds = [get_answer_value(states[i]["answer"]) for i in range(len(states))] + acc = np.mean(np.array(preds) == np.array(labels)) + invalid = np.mean(np.array(preds) == INVALID) + + return float(acc), float(invalid), float(latency) + + +class TestDeepSeekR1EvalAMD(unittest.TestCase): + """DeepSeek-R1 GSM8K Completion Evaluation Test for AMD MI300X.""" + + @classmethod + def setUpClass(cls): + cls.models = DEEPSEEK_R1_MODELS + cls.base_url = DEFAULT_URL_FOR_TEST + cls.num_questions = int(os.environ.get("GSM8K_NUM_QUESTIONS", "200")) + + def test_deepseek_r1_accuracy(self): + """Test DeepSeek-R1 models with GSM8K completion benchmark.""" + all_results = [] + summary = "### DeepSeek-R1 Models (MI300X)\n\n" + summary += "| Model | Variant | TP | Accuracy | Threshold | Status |\n" + summary += "| ----- | ------- | -- | -------- | --------- | ------ |\n" + + for config in self.models: + display_name = config.get_display_name() + with self.subTest(model=display_name): + print(f"\n{'='*60}") + print(f"Testing: {display_name}") + print(f"{'='*60}") + + env = os.environ.copy() + for key, value in config.env_vars.items(): + env[key] = value + + other_args = list(config.other_args) + other_args.extend(["--tp", str(config.tp_size)]) + timeout = config.timeout or DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH + + try: + process = popen_launch_server( + model=config.model_path, + base_url=self.base_url, + timeout=timeout, + other_args=other_args, + env=env, + ) + + try: + acc, invalid, latency = run_gsm8k_benchmark( + self.base_url, num_questions=self.num_questions + ) + passed = acc >= config.accuracy_threshold + status = "✅ PASS" if passed else "❌ FAIL" + + all_results.append( + { + "model": display_name, + "accuracy": acc, + "passed": passed, + } + ) + summary += f"| {config.model_path} | {config.variant or 'N/A'} | {config.tp_size} | {acc:.3f} | {config.accuracy_threshold} | {status} |\n" + + finally: + kill_process_tree(process.pid) + + except Exception as e: + summary += f"| {config.model_path} | {config.variant or 'N/A'} | {config.tp_size} | N/A | {config.accuracy_threshold} | ❌ ERROR |\n" + all_results.append( + { + "model": display_name, + "accuracy": None, + "passed": False, + "error": str(e), + } + ) + + if is_in_ci(): + write_github_step_summary(summary) + + failed = [r for r in all_results if not r["passed"]] + if failed: + raise AssertionError(f"Failed models: {[r['model'] for r in failed]}") + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/amd/accuracy/test_deepseek_v31_eval_amd.py b/test/registered/amd/accuracy/test_deepseek_v31_eval_amd.py new file mode 100644 index 000000000..e50e95426 --- /dev/null +++ b/test/registered/amd/accuracy/test_deepseek_v31_eval_amd.py @@ -0,0 +1,159 @@ +"""AMD DeepSeek-V3.1 GSM8K Completion Evaluation Test (8-GPU) + +Tests DeepSeek-V3.1 model using few-shot completion benchmark on MI300X. + +Registry: nightly-amd-8-gpu-deepseek-v31 suite +""" + +import ast +import os +import re +import time +import unittest + +import numpy as np + +from sglang.srt.utils import kill_process_tree +from sglang.test.ci.ci_register import register_amd_ci +from sglang.test.test_utils import ( + DEFAULT_URL_FOR_TEST, + is_in_ci, + popen_launch_server, + write_github_step_summary, +) +from sglang.utils import download_and_cache_file, read_jsonl + +# Register for AMD CI - DeepSeek-V3.1 accuracy tests (~60 min) +register_amd_ci( + est_time=3600, suite="nightly-amd-accuracy-8-gpu-deepseek-v31", nightly=True +) + +INVALID = -9999999 + +DEEPSEEK_V31_MODEL_PATH = os.environ.get( + "DEEPSEEK_V31_MODEL_PATH", "deepseek-ai/DeepSeek-V3-0324" +) + + +def get_one_example(lines, i, include_answer): + ret = "Question: " + lines[i]["question"] + "\nAnswer:" + if include_answer: + ret += " " + lines[i]["answer"] + return ret + + +def get_few_shot_examples(lines, k): + ret = "" + for i in range(k): + ret += get_one_example(lines, i, True) + "\n\n" + return ret + + +def get_answer_value(answer_str): + answer_str = answer_str.replace(",", "") + numbers = re.findall(r"\d+", answer_str) + if len(numbers) < 1: + return INVALID + try: + return ast.literal_eval(numbers[-1]) + except SyntaxError: + return INVALID + + +def run_gsm8k_benchmark(base_url, num_questions=200, num_shots=5, parallel=64): + import sglang as sgl + from sglang.lang.backend.runtime_endpoint import RuntimeEndpoint + + url = "https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/test.jsonl" + data_path = download_and_cache_file(url) + lines = list(read_jsonl(data_path)) + + few_shot_examples = get_few_shot_examples(lines, num_shots) + questions = [] + labels = [] + for i in range(len(lines[:num_questions])): + questions.append(get_one_example(lines, i, False)) + labels.append(get_answer_value(lines[i]["answer"])) + arguments = [{"question": q} for q in questions] + + @sgl.function + def few_shot_gsm8k(s, question): + s += few_shot_examples + question + s += sgl.gen( + "answer", max_tokens=512, stop=["Question", "Assistant:", "<|separator|>"] + ) + + backend = RuntimeEndpoint(base_url) + sgl.set_default_backend(backend) + + tic = time.perf_counter() + states = few_shot_gsm8k.run_batch( + arguments, temperature=0, num_threads=parallel, progress_bar=True + ) + latency = time.perf_counter() - tic + + preds = [get_answer_value(states[i]["answer"]) for i in range(len(states))] + acc = np.mean(np.array(preds) == np.array(labels)) + return float(acc), float(latency) + + +class TestDeepSeekV31EvalAMD(unittest.TestCase): + """DeepSeek-V3.1 GSM8K Completion Evaluation Test for AMD MI300X.""" + + @classmethod + def setUpClass(cls): + cls.base_url = DEFAULT_URL_FOR_TEST + cls.num_questions = int(os.environ.get("GSM8K_NUM_QUESTIONS", "200")) + cls.accuracy_threshold = 0.90 + + def test_deepseek_v31_accuracy(self): + """Test DeepSeek-V3.1 with GSM8K completion benchmark.""" + env = os.environ.copy() + env["SGLANG_USE_AITER"] = "1" + + other_args = [ + "--tp", + "8", + "--attention-backend", + "aiter", + "--chunked-prefill-size", + "131072", + "--mem-fraction-static", + "0.85", + "--trust-remote-code", + ] + + process = popen_launch_server( + model=DEEPSEEK_V31_MODEL_PATH, + base_url=self.base_url, + timeout=3600, + other_args=other_args, + env=env, + ) + + try: + acc, latency = run_gsm8k_benchmark( + self.base_url, num_questions=self.num_questions + ) + passed = acc >= self.accuracy_threshold + status = "✅ PASS" if passed else "❌ FAIL" + + summary = f"### DeepSeek-V3.1 (MI300X)\n\n" + summary += f"| Model | Accuracy | Threshold | Status |\n" + summary += f"| ----- | -------- | --------- | ------ |\n" + summary += f"| {DEEPSEEK_V31_MODEL_PATH} | {acc:.3f} | {self.accuracy_threshold} | {status} |\n" + + if is_in_ci(): + write_github_step_summary(summary) + + self.assertGreaterEqual( + acc, + self.accuracy_threshold, + f"Accuracy {acc:.3f} below threshold {self.accuracy_threshold}", + ) + finally: + kill_process_tree(process.pid) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/amd/accuracy/test_gpt_oss_eval_amd.py b/test/registered/amd/accuracy/test_gpt_oss_eval_amd.py new file mode 100644 index 000000000..b736361ff --- /dev/null +++ b/test/registered/amd/accuracy/test_gpt_oss_eval_amd.py @@ -0,0 +1,247 @@ +"""AMD GPT-OSS GSM8K Completion Evaluation Test (8-GPU) + +Tests GPT-OSS models (lmsys/gpt-oss-20b-bf16, lmsys/gpt-oss-120b-bf16) using +few-shot completion benchmark on MI300X. + +Registry: nightly-amd-8-gpu suite +""" + +import ast +import os +import re +import time +import unittest +from dataclasses import dataclass +from typing import List, Optional, Tuple + +import numpy as np + +from sglang.srt.utils import kill_process_tree +from sglang.test.ci.ci_register import register_amd_ci +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + is_in_ci, + popen_launch_server, + write_github_step_summary, +) +from sglang.utils import download_and_cache_file, read_jsonl + +# Register for AMD CI - GPT-OSS accuracy tests (~30 min) +register_amd_ci(est_time=1800, suite="nightly-amd-accuracy-8-gpu-gpt-oss", nightly=True) + +INVALID = -9999999 + + +@dataclass +class ModelConfig: + """Configuration for a model to test.""" + + model_path: str + tp_size: int = 8 + accuracy_threshold: float = 0.50 + other_args: Optional[List[str]] = None + env_vars: Optional[dict] = None + timeout: Optional[int] = None + + def __post_init__(self): + if self.other_args is None: + self.other_args = [] + if self.env_vars is None: + self.env_vars = {} + + +# GPT-OSS models for MI300X +GPT_OSS_MODELS = [ + ModelConfig( + model_path="lmsys/gpt-oss-20b-bf16", + tp_size=8, + accuracy_threshold=0.47, + other_args=[ + "--chunked-prefill-size", + "130172", + "--max-running-requests", + "128", + "--mem-fraction-static", + "0.85", + "--attention-backend", + "triton", + "--trust-remote-code", + ], + env_vars={"SGLANG_USE_AITER": "0"}, + ), + ModelConfig( + model_path="lmsys/gpt-oss-120b-bf16", + tp_size=8, + accuracy_threshold=0.79, + timeout=900, + other_args=[ + "--chunked-prefill-size", + "130172", + "--max-running-requests", + "128", + "--mem-fraction-static", + "0.85", + "--attention-backend", + "triton", + "--trust-remote-code", + ], + env_vars={"SGLANG_USE_AITER": "0"}, + ), +] + + +def get_one_example(lines, i, include_answer): + """Format a single GSM8K example.""" + ret = "Question: " + lines[i]["question"] + "\nAnswer:" + if include_answer: + ret += " " + lines[i]["answer"] + return ret + + +def get_few_shot_examples(lines, k): + """Get k few-shot examples for prompting.""" + ret = "" + for i in range(k): + ret += get_one_example(lines, i, True) + "\n\n" + return ret + + +def get_answer_value(answer_str): + """Extract numerical answer from response.""" + answer_str = answer_str.replace(",", "") + numbers = re.findall(r"\d+", answer_str) + if len(numbers) < 1: + return INVALID + try: + return ast.literal_eval(numbers[-1]) + except SyntaxError: + return INVALID + + +def run_gsm8k_benchmark( + base_url: str, + num_questions: int = 200, + num_shots: int = 5, + parallel: int = 64, +) -> Tuple[float, float, float]: + """Run GSM8K few-shot completion benchmark.""" + import sglang as sgl + from sglang.lang.backend.runtime_endpoint import RuntimeEndpoint + + url = "https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/test.jsonl" + data_path = download_and_cache_file(url) + lines = list(read_jsonl(data_path)) + + few_shot_examples = get_few_shot_examples(lines, num_shots) + + questions = [] + labels = [] + for i in range(len(lines[:num_questions])): + questions.append(get_one_example(lines, i, False)) + labels.append(get_answer_value(lines[i]["answer"])) + assert all(l != INVALID for l in labels) + arguments = [{"question": q} for q in questions] + + @sgl.function + def few_shot_gsm8k(s, question): + s += few_shot_examples + question + s += sgl.gen( + "answer", max_tokens=512, stop=["Question", "Assistant:", "<|separator|>"] + ) + + backend = RuntimeEndpoint(base_url) + sgl.set_default_backend(backend) + + tic = time.perf_counter() + states = few_shot_gsm8k.run_batch( + arguments, temperature=0, num_threads=parallel, progress_bar=True + ) + latency = time.perf_counter() - tic + + preds = [get_answer_value(states[i]["answer"]) for i in range(len(states))] + acc = np.mean(np.array(preds) == np.array(labels)) + invalid = np.mean(np.array(preds) == INVALID) + + return float(acc), float(invalid), float(latency) + + +class TestGptOssEvalAMD(unittest.TestCase): + """GPT-OSS GSM8K Completion Evaluation Test for AMD MI300X.""" + + @classmethod + def setUpClass(cls): + cls.models = GPT_OSS_MODELS + cls.base_url = DEFAULT_URL_FOR_TEST + cls.num_questions = int(os.environ.get("GSM8K_NUM_QUESTIONS", "200")) + + def test_gpt_oss_accuracy(self): + """Test GPT-OSS models with GSM8K completion benchmark.""" + all_results = [] + summary = "### GPT-OSS Models (MI300X)\n\n" + summary += "| Model | TP | Accuracy | Threshold | Status |\n" + summary += "| ----- | -- | -------- | --------- | ------ |\n" + + for config in self.models: + with self.subTest(model=config.model_path): + print(f"\n{'='*60}") + print(f"Testing: {config.model_path}") + print(f"{'='*60}") + + env = os.environ.copy() + for key, value in config.env_vars.items(): + env[key] = value + + other_args = list(config.other_args) + other_args.extend(["--tp", str(config.tp_size)]) + timeout = config.timeout or DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH + + try: + process = popen_launch_server( + model=config.model_path, + base_url=self.base_url, + timeout=timeout, + other_args=other_args, + env=env, + ) + + try: + acc, invalid, latency = run_gsm8k_benchmark( + self.base_url, num_questions=self.num_questions + ) + passed = acc >= config.accuracy_threshold + status = "✅ PASS" if passed else "❌ FAIL" + + all_results.append( + { + "model": config.model_path, + "accuracy": acc, + "passed": passed, + } + ) + summary += f"| {config.model_path} | {config.tp_size} | {acc:.3f} | {config.accuracy_threshold} | {status} |\n" + + finally: + kill_process_tree(process.pid) + + except Exception as e: + summary += f"| {config.model_path} | {config.tp_size} | N/A | {config.accuracy_threshold} | ❌ ERROR |\n" + all_results.append( + { + "model": config.model_path, + "accuracy": None, + "passed": False, + "error": str(e), + } + ) + + if is_in_ci(): + write_github_step_summary(summary) + + failed = [r for r in all_results if not r["passed"]] + if failed: + raise AssertionError(f"Failed models: {[r['model'] for r in failed]}") + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/amd/accuracy/test_grok1_fp8_eval_amd.py b/test/registered/amd/accuracy/test_grok1_fp8_eval_amd.py new file mode 100644 index 000000000..9b9431e6a --- /dev/null +++ b/test/registered/amd/accuracy/test_grok1_fp8_eval_amd.py @@ -0,0 +1,162 @@ +"""AMD GROK1-FP8 GSM8K Completion Evaluation Test (8-GPU) + +Tests Grok-1 FP8 model using few-shot completion benchmark on MI300X. + +Registry: nightly-amd-8-gpu-grok1-fp8 suite +""" + +import ast +import os +import re +import time +import unittest + +import numpy as np + +from sglang.srt.utils import kill_process_tree +from sglang.test.ci.ci_register import register_amd_ci +from sglang.test.test_utils import ( + DEFAULT_URL_FOR_TEST, + is_in_ci, + popen_launch_server, + write_github_step_summary, +) +from sglang.utils import download_and_cache_file, read_jsonl + +# Register for AMD CI - GROK1-FP8 accuracy tests (~25 min) +register_amd_ci( + est_time=1500, suite="nightly-amd-accuracy-8-gpu-grok1-fp8", nightly=True +) + +INVALID = -9999999 + +GROK1_FP8_MODEL_PATH = os.environ.get("GROK1_MODEL_PATH", "lmzheng/grok-1") +GROK1_TOKENIZER_PATH = os.environ.get("GROK1_TOKENIZER_PATH", "Xenova/grok-1-tokenizer") + + +def get_one_example(lines, i, include_answer): + ret = "Question: " + lines[i]["question"] + "\nAnswer:" + if include_answer: + ret += " " + lines[i]["answer"] + return ret + + +def get_few_shot_examples(lines, k): + ret = "" + for i in range(k): + ret += get_one_example(lines, i, True) + "\n\n" + return ret + + +def get_answer_value(answer_str): + answer_str = answer_str.replace(",", "") + numbers = re.findall(r"\d+", answer_str) + if len(numbers) < 1: + return INVALID + try: + return ast.literal_eval(numbers[-1]) + except SyntaxError: + return INVALID + + +def run_gsm8k_benchmark(base_url, num_questions=200, num_shots=5, parallel=64): + import sglang as sgl + from sglang.lang.backend.runtime_endpoint import RuntimeEndpoint + + url = "https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/test.jsonl" + data_path = download_and_cache_file(url) + lines = list(read_jsonl(data_path)) + + few_shot_examples = get_few_shot_examples(lines, num_shots) + questions = [] + labels = [] + for i in range(len(lines[:num_questions])): + questions.append(get_one_example(lines, i, False)) + labels.append(get_answer_value(lines[i]["answer"])) + arguments = [{"question": q} for q in questions] + + @sgl.function + def few_shot_gsm8k(s, question): + s += few_shot_examples + question + s += sgl.gen( + "answer", max_tokens=512, stop=["Question", "Assistant:", "<|separator|>"] + ) + + backend = RuntimeEndpoint(base_url) + sgl.set_default_backend(backend) + + tic = time.perf_counter() + states = few_shot_gsm8k.run_batch( + arguments, temperature=0, num_threads=parallel, progress_bar=True + ) + latency = time.perf_counter() - tic + + preds = [get_answer_value(states[i]["answer"]) for i in range(len(states))] + acc = np.mean(np.array(preds) == np.array(labels)) + return float(acc), float(latency) + + +class TestGrok1FP8EvalAMD(unittest.TestCase): + """GROK1-FP8 GSM8K Completion Evaluation Test for AMD MI300X.""" + + @classmethod + def setUpClass(cls): + cls.base_url = DEFAULT_URL_FOR_TEST + cls.num_questions = int(os.environ.get("GSM8K_NUM_QUESTIONS", "200")) + cls.accuracy_threshold = 0.80 + + def test_grok1_fp8_accuracy(self): + """Test Grok-1 FP8 with GSM8K completion benchmark.""" + env = os.environ.copy() + env["RCCL_MSCCL_ENABLE"] = "0" + env["SGLANG_USE_AITER"] = "1" + env["SGLANG_INT4_WEIGHT"] = "0" + + other_args = [ + "--tp", + "8", + "--quantization", + "fp8", + "--attention-backend", + "aiter", + "--mem-fraction-static", + "0.85", + "--tokenizer-path", + GROK1_TOKENIZER_PATH, + "--trust-remote-code", + ] + + process = popen_launch_server( + model=GROK1_FP8_MODEL_PATH, + base_url=self.base_url, + timeout=3600, + other_args=other_args, + env=env, + ) + + try: + acc, latency = run_gsm8k_benchmark( + self.base_url, num_questions=self.num_questions + ) + passed = acc >= self.accuracy_threshold + status = "✅ PASS" if passed else "❌ FAIL" + + summary = f"### GROK1-FP8 (MI300X)\n\n" + summary += f"| Model | Accuracy | Threshold | Status |\n" + summary += f"| ----- | -------- | --------- | ------ |\n" + summary += f"| {GROK1_FP8_MODEL_PATH} | {acc:.3f} | {self.accuracy_threshold} | {status} |\n" + + if is_in_ci(): + write_github_step_summary(summary) + + self.assertGreaterEqual( + acc, + self.accuracy_threshold, + f"Accuracy {acc:.3f} below threshold {self.accuracy_threshold}", + ) + finally: + kill_process_tree(process.pid) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/amd/accuracy/test_grok1_int4_eval_amd.py b/test/registered/amd/accuracy/test_grok1_int4_eval_amd.py new file mode 100644 index 000000000..539f4754b --- /dev/null +++ b/test/registered/amd/accuracy/test_grok1_int4_eval_amd.py @@ -0,0 +1,162 @@ +"""AMD GROK1-INT4 GSM8K Completion Evaluation Test (8-GPU) + +Tests Grok-1 INT4 (W4A8KV8) model using few-shot completion benchmark on MI300X. + +Registry: nightly-amd-8-gpu-grok1-int4 suite +""" + +import ast +import os +import re +import time +import unittest + +import numpy as np + +from sglang.srt.utils import kill_process_tree +from sglang.test.ci.ci_register import register_amd_ci +from sglang.test.test_utils import ( + DEFAULT_URL_FOR_TEST, + is_in_ci, + popen_launch_server, + write_github_step_summary, +) +from sglang.utils import download_and_cache_file, read_jsonl + +# Register for AMD CI - GROK1-INT4 accuracy tests (~25 min) +register_amd_ci( + est_time=1500, suite="nightly-amd-accuracy-8-gpu-grok1-int4", nightly=True +) + +INVALID = -9999999 + +GROK1_INT4_MODEL_PATH = os.environ.get("GROK1_INT4_MODEL_PATH", "amd/grok-1-W4A8KV8") +GROK1_TOKENIZER_PATH = os.environ.get("GROK1_TOKENIZER_PATH", "Xenova/grok-1-tokenizer") + + +def get_one_example(lines, i, include_answer): + ret = "Question: " + lines[i]["question"] + "\nAnswer:" + if include_answer: + ret += " " + lines[i]["answer"] + return ret + + +def get_few_shot_examples(lines, k): + ret = "" + for i in range(k): + ret += get_one_example(lines, i, True) + "\n\n" + return ret + + +def get_answer_value(answer_str): + answer_str = answer_str.replace(",", "") + numbers = re.findall(r"\d+", answer_str) + if len(numbers) < 1: + return INVALID + try: + return ast.literal_eval(numbers[-1]) + except SyntaxError: + return INVALID + + +def run_gsm8k_benchmark(base_url, num_questions=200, num_shots=5, parallel=64): + import sglang as sgl + from sglang.lang.backend.runtime_endpoint import RuntimeEndpoint + + url = "https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/test.jsonl" + data_path = download_and_cache_file(url) + lines = list(read_jsonl(data_path)) + + few_shot_examples = get_few_shot_examples(lines, num_shots) + questions = [] + labels = [] + for i in range(len(lines[:num_questions])): + questions.append(get_one_example(lines, i, False)) + labels.append(get_answer_value(lines[i]["answer"])) + arguments = [{"question": q} for q in questions] + + @sgl.function + def few_shot_gsm8k(s, question): + s += few_shot_examples + question + s += sgl.gen( + "answer", max_tokens=512, stop=["Question", "Assistant:", "<|separator|>"] + ) + + backend = RuntimeEndpoint(base_url) + sgl.set_default_backend(backend) + + tic = time.perf_counter() + states = few_shot_gsm8k.run_batch( + arguments, temperature=0, num_threads=parallel, progress_bar=True + ) + latency = time.perf_counter() - tic + + preds = [get_answer_value(states[i]["answer"]) for i in range(len(states))] + acc = np.mean(np.array(preds) == np.array(labels)) + return float(acc), float(latency) + + +class TestGrok1INT4EvalAMD(unittest.TestCase): + """GROK1-INT4 GSM8K Completion Evaluation Test for AMD MI300X.""" + + @classmethod + def setUpClass(cls): + cls.base_url = DEFAULT_URL_FOR_TEST + cls.num_questions = int(os.environ.get("GSM8K_NUM_QUESTIONS", "200")) + cls.accuracy_threshold = 0.80 + + def test_grok1_int4_accuracy(self): + """Test Grok-1 INT4 with GSM8K completion benchmark.""" + env = os.environ.copy() + env["RCCL_MSCCL_ENABLE"] = "0" + env["SGLANG_USE_AITER"] = "1" + env["SGLANG_INT4_WEIGHT"] = "1" + + other_args = [ + "--tp", + "8", + "--quantization", + "fp8", + "--attention-backend", + "aiter", + "--mem-fraction-static", + "0.85", + "--tokenizer-path", + GROK1_TOKENIZER_PATH, + "--trust-remote-code", + ] + + process = popen_launch_server( + model=GROK1_INT4_MODEL_PATH, + base_url=self.base_url, + timeout=3600, + other_args=other_args, + env=env, + ) + + try: + acc, latency = run_gsm8k_benchmark( + self.base_url, num_questions=self.num_questions + ) + passed = acc >= self.accuracy_threshold + status = "✅ PASS" if passed else "❌ FAIL" + + summary = f"### GROK1-INT4 (MI300X)\n\n" + summary += f"| Model | Accuracy | Threshold | Status |\n" + summary += f"| ----- | -------- | --------- | ------ |\n" + summary += f"| {GROK1_INT4_MODEL_PATH} | {acc:.3f} | {self.accuracy_threshold} | {status} |\n" + + if is_in_ci(): + write_github_step_summary(summary) + + self.assertGreaterEqual( + acc, + self.accuracy_threshold, + f"Accuracy {acc:.3f} below threshold {self.accuracy_threshold}", + ) + finally: + kill_process_tree(process.pid) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/amd/accuracy/test_grok2_eval_amd.py b/test/registered/amd/accuracy/test_grok2_eval_amd.py new file mode 100644 index 000000000..28610c655 --- /dev/null +++ b/test/registered/amd/accuracy/test_grok2_eval_amd.py @@ -0,0 +1,162 @@ +"""AMD GROK2 GSM8K Completion Evaluation Test (8-GPU) + +Tests Grok-2 model using few-shot completion benchmark on MI300X. + +Registry: nightly-amd-8-gpu-grok2 suite +""" + +import ast +import os +import re +import time +import unittest + +import numpy as np + +from sglang.srt.utils import kill_process_tree +from sglang.test.ci.ci_register import register_amd_ci +from sglang.test.test_utils import ( + DEFAULT_URL_FOR_TEST, + is_in_ci, + popen_launch_server, + write_github_step_summary, +) +from sglang.utils import download_and_cache_file, read_jsonl + +# Register for AMD CI - GROK2 accuracy tests (~25 min) +register_amd_ci(est_time=1500, suite="nightly-amd-accuracy-8-gpu-grok2", nightly=True) + +INVALID = -9999999 + +GROK2_MODEL_PATH = os.environ.get("GROK2_MODEL_PATH", "xai-org/grok-2") +GROK2_TOKENIZER_PATH = os.environ.get( + "GROK2_TOKENIZER_PATH", "alvarobartt/grok-2-tokenizer" +) + + +def get_one_example(lines, i, include_answer): + ret = "Question: " + lines[i]["question"] + "\nAnswer:" + if include_answer: + ret += " " + lines[i]["answer"] + return ret + + +def get_few_shot_examples(lines, k): + ret = "" + for i in range(k): + ret += get_one_example(lines, i, True) + "\n\n" + return ret + + +def get_answer_value(answer_str): + answer_str = answer_str.replace(",", "") + numbers = re.findall(r"\d+", answer_str) + if len(numbers) < 1: + return INVALID + try: + return ast.literal_eval(numbers[-1]) + except SyntaxError: + return INVALID + + +def run_gsm8k_benchmark(base_url, num_questions=200, num_shots=5, parallel=64): + import sglang as sgl + from sglang.lang.backend.runtime_endpoint import RuntimeEndpoint + + url = "https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/test.jsonl" + data_path = download_and_cache_file(url) + lines = list(read_jsonl(data_path)) + + few_shot_examples = get_few_shot_examples(lines, num_shots) + questions = [] + labels = [] + for i in range(len(lines[:num_questions])): + questions.append(get_one_example(lines, i, False)) + labels.append(get_answer_value(lines[i]["answer"])) + arguments = [{"question": q} for q in questions] + + @sgl.function + def few_shot_gsm8k(s, question): + s += few_shot_examples + question + s += sgl.gen( + "answer", max_tokens=512, stop=["Question", "Assistant:", "<|separator|>"] + ) + + backend = RuntimeEndpoint(base_url) + sgl.set_default_backend(backend) + + tic = time.perf_counter() + states = few_shot_gsm8k.run_batch( + arguments, temperature=0, num_threads=parallel, progress_bar=True + ) + latency = time.perf_counter() - tic + + preds = [get_answer_value(states[i]["answer"]) for i in range(len(states))] + acc = np.mean(np.array(preds) == np.array(labels)) + return float(acc), float(latency) + + +class TestGrok2EvalAMD(unittest.TestCase): + """GROK2 GSM8K Completion Evaluation Test for AMD MI300X.""" + + @classmethod + def setUpClass(cls): + cls.base_url = DEFAULT_URL_FOR_TEST + cls.num_questions = int(os.environ.get("GSM8K_NUM_QUESTIONS", "200")) + cls.accuracy_threshold = 0.915 + + def test_grok2_accuracy(self): + """Test Grok-2 with GSM8K completion benchmark.""" + env = os.environ.copy() + env["RCCL_MSCCL_ENABLE"] = "0" + env["SGLANG_USE_AITER"] = "1" + env["SGLANG_INT4_WEIGHT"] = "0" + + other_args = [ + "--tp", + "8", + "--quantization", + "fp8", + "--attention-backend", + "aiter", + "--mem-fraction-static", + "0.85", + "--tokenizer-path", + GROK2_TOKENIZER_PATH, + "--trust-remote-code", + ] + + process = popen_launch_server( + model=GROK2_MODEL_PATH, + base_url=self.base_url, + timeout=3600, + other_args=other_args, + env=env, + ) + + try: + acc, latency = run_gsm8k_benchmark( + self.base_url, num_questions=self.num_questions + ) + passed = acc >= self.accuracy_threshold + status = "✅ PASS" if passed else "❌ FAIL" + + summary = f"### GROK2 (MI300X)\n\n" + summary += f"| Model | Accuracy | Threshold | Status |\n" + summary += f"| ----- | -------- | --------- | ------ |\n" + summary += f"| {GROK2_MODEL_PATH} | {acc:.3f} | {self.accuracy_threshold} | {status} |\n" + + if is_in_ci(): + write_github_step_summary(summary) + + self.assertGreaterEqual( + acc, + self.accuracy_threshold, + f"Accuracy {acc:.3f} below threshold {self.accuracy_threshold}", + ) + finally: + kill_process_tree(process.pid) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/amd/accuracy/test_grok_eval_amd.py b/test/registered/amd/accuracy/test_grok_eval_amd.py new file mode 100644 index 000000000..8ad217349 --- /dev/null +++ b/test/registered/amd/accuracy/test_grok_eval_amd.py @@ -0,0 +1,287 @@ +"""AMD GROK GSM8K Completion Evaluation Test (8-GPU) + +Tests GROK models (Grok-1 FP8, Grok-1 INT4, Grok-2) using +few-shot completion benchmark on MI300X. + +Registry: nightly-amd-8-gpu-grok suite +""" + +import ast +import os +import re +import time +import unittest +from dataclasses import dataclass +from typing import List, Optional, Tuple + +import numpy as np + +from sglang.srt.utils import kill_process_tree +from sglang.test.ci.ci_register import register_amd_ci +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + is_in_ci, + popen_launch_server, + write_github_step_summary, +) +from sglang.utils import download_and_cache_file, read_jsonl + +# DISABLED: Split into individual files for each model variant +# See: test_grok1_fp8_eval_amd.py, test_grok1_int4_eval_amd.py, test_grok2_eval_amd.py +register_amd_ci( + est_time=2700, + suite="nightly-amd-8-gpu-grok", + nightly=True, + disabled="Split into test_grok1_fp8_eval_amd.py, test_grok1_int4_eval_amd.py, test_grok2_eval_amd.py", +) + +INVALID = -9999999 + + +@dataclass +class ModelConfig: + """Configuration for a model to test.""" + + model_path: str + tp_size: int = 8 + accuracy_threshold: float = 0.50 + other_args: Optional[List[str]] = None + env_vars: Optional[dict] = None + tokenizer_path: Optional[str] = None + timeout: Optional[int] = None + + def __post_init__(self): + if self.other_args is None: + self.other_args = [] + if self.env_vars is None: + self.env_vars = {} + + +# GROK models for MI300X +GROK_MODELS = [ + # GROK1-FP8 + ModelConfig( + model_path="lmzheng/grok-1", + tp_size=8, + accuracy_threshold=0.80, + timeout=3600, + tokenizer_path="Xenova/grok-1-tokenizer", + other_args=[ + "--quantization", + "fp8", + "--attention-backend", + "aiter", + "--mem-fraction-static", + "0.85", + "--trust-remote-code", + ], + env_vars={ + "RCCL_MSCCL_ENABLE": "0", + "SGLANG_USE_AITER": "1", + "SGLANG_INT4_WEIGHT": "0", + }, + ), + # GROK1-INT4 + ModelConfig( + model_path="amd/grok-1-W4A8KV8", + tp_size=8, + accuracy_threshold=0.80, + timeout=3600, + tokenizer_path="Xenova/grok-1-tokenizer", + other_args=[ + "--quantization", + "fp8", + "--attention-backend", + "aiter", + "--mem-fraction-static", + "0.85", + "--trust-remote-code", + ], + env_vars={ + "RCCL_MSCCL_ENABLE": "0", + "SGLANG_USE_AITER": "1", + "SGLANG_INT4_WEIGHT": "1", + }, + ), + # GROK2 + ModelConfig( + model_path="xai-org/grok-2", + tp_size=8, + accuracy_threshold=0.915, + timeout=3600, + tokenizer_path="alvarobartt/grok-2-tokenizer", + other_args=[ + "--quantization", + "fp8", + "--attention-backend", + "aiter", + "--mem-fraction-static", + "0.85", + "--trust-remote-code", + ], + env_vars={ + "RCCL_MSCCL_ENABLE": "0", + "SGLANG_USE_AITER": "1", + "SGLANG_INT4_WEIGHT": "0", + }, + ), +] + + +def get_one_example(lines, i, include_answer): + """Format a single GSM8K example.""" + ret = "Question: " + lines[i]["question"] + "\nAnswer:" + if include_answer: + ret += " " + lines[i]["answer"] + return ret + + +def get_few_shot_examples(lines, k): + """Get k few-shot examples for prompting.""" + ret = "" + for i in range(k): + ret += get_one_example(lines, i, True) + "\n\n" + return ret + + +def get_answer_value(answer_str): + """Extract numerical answer from response.""" + answer_str = answer_str.replace(",", "") + numbers = re.findall(r"\d+", answer_str) + if len(numbers) < 1: + return INVALID + try: + return ast.literal_eval(numbers[-1]) + except SyntaxError: + return INVALID + + +def run_gsm8k_benchmark( + base_url: str, + num_questions: int = 200, + num_shots: int = 5, + parallel: int = 64, +) -> Tuple[float, float, float]: + """Run GSM8K few-shot completion benchmark.""" + import sglang as sgl + from sglang.lang.backend.runtime_endpoint import RuntimeEndpoint + + url = "https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/test.jsonl" + data_path = download_and_cache_file(url) + lines = list(read_jsonl(data_path)) + + few_shot_examples = get_few_shot_examples(lines, num_shots) + + questions = [] + labels = [] + for i in range(len(lines[:num_questions])): + questions.append(get_one_example(lines, i, False)) + labels.append(get_answer_value(lines[i]["answer"])) + assert all(l != INVALID for l in labels) + arguments = [{"question": q} for q in questions] + + @sgl.function + def few_shot_gsm8k(s, question): + s += few_shot_examples + question + s += sgl.gen( + "answer", max_tokens=512, stop=["Question", "Assistant:", "<|separator|>"] + ) + + backend = RuntimeEndpoint(base_url) + sgl.set_default_backend(backend) + + tic = time.perf_counter() + states = few_shot_gsm8k.run_batch( + arguments, temperature=0, num_threads=parallel, progress_bar=True + ) + latency = time.perf_counter() - tic + + preds = [get_answer_value(states[i]["answer"]) for i in range(len(states))] + acc = np.mean(np.array(preds) == np.array(labels)) + invalid = np.mean(np.array(preds) == INVALID) + + return float(acc), float(invalid), float(latency) + + +class TestGrokEvalAMD(unittest.TestCase): + """GROK GSM8K Completion Evaluation Test for AMD MI300X.""" + + @classmethod + def setUpClass(cls): + cls.models = GROK_MODELS + cls.base_url = DEFAULT_URL_FOR_TEST + cls.num_questions = int(os.environ.get("GSM8K_NUM_QUESTIONS", "200")) + + def test_grok_accuracy(self): + """Test GROK models with GSM8K completion benchmark.""" + all_results = [] + summary = "### GROK Models (MI300X)\n\n" + summary += "| Model | TP | Accuracy | Threshold | Status |\n" + summary += "| ----- | -- | -------- | --------- | ------ |\n" + + for config in self.models: + with self.subTest(model=config.model_path): + print(f"\n{'='*60}") + print(f"Testing: {config.model_path}") + print(f"{'='*60}") + + env = os.environ.copy() + for key, value in config.env_vars.items(): + env[key] = value + + other_args = list(config.other_args) + other_args.extend(["--tp", str(config.tp_size)]) + if config.tokenizer_path: + other_args.extend(["--tokenizer-path", config.tokenizer_path]) + timeout = config.timeout or DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH + + try: + process = popen_launch_server( + model=config.model_path, + base_url=self.base_url, + timeout=timeout, + other_args=other_args, + env=env, + ) + + try: + acc, invalid, latency = run_gsm8k_benchmark( + self.base_url, num_questions=self.num_questions + ) + passed = acc >= config.accuracy_threshold + status = "✅ PASS" if passed else "❌ FAIL" + + all_results.append( + { + "model": config.model_path, + "accuracy": acc, + "passed": passed, + } + ) + summary += f"| {config.model_path} | {config.tp_size} | {acc:.3f} | {config.accuracy_threshold} | {status} |\n" + + finally: + kill_process_tree(process.pid) + + except Exception as e: + summary += f"| {config.model_path} | {config.tp_size} | N/A | {config.accuracy_threshold} | ❌ ERROR |\n" + all_results.append( + { + "model": config.model_path, + "accuracy": None, + "passed": False, + "error": str(e), + } + ) + + if is_in_ci(): + write_github_step_summary(summary) + + failed = [r for r in all_results if not r["passed"]] + if failed: + raise AssertionError(f"Failed models: {[r['model'] for r in failed]}") + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/amd/nightly/test_vlms_mmmu_eval_amd.py b/test/registered/amd/accuracy/test_vlms_mmmu_eval_amd.py similarity index 98% rename from test/registered/amd/nightly/test_vlms_mmmu_eval_amd.py rename to test/registered/amd/accuracy/test_vlms_mmmu_eval_amd.py index d9438e6ca..7909120a3 100644 --- a/test/registered/amd/nightly/test_vlms_mmmu_eval_amd.py +++ b/test/registered/amd/accuracy/test_vlms_mmmu_eval_amd.py @@ -1,5 +1,5 @@ """ -AMD VLM MMMU Evaluation Test (Migrated from test/srt/nightly/) +AMD VLM MMMU Evaluation Test - MI30x Only This test evaluates Vision-Language Models (VLMs) on the MMMU benchmark on AMD GPUs. Models are selected based on compatibility with AMD/ROCm platform. @@ -11,6 +11,7 @@ VLMs tested here: - deepseek-vl2-small Note: Some VLMs from the Nvidia test are excluded due to AMD compatibility issues. +Note: This test runs only on MI30x runners (linux-mi325-gpu-2), not on MI35x. Registry: nightly-amd-vlm suite (2-GPU VLM tests) """ @@ -34,7 +35,7 @@ from sglang.test.test_utils import ( ) # Register for AMD CI - VLM MMMU evaluation tests (~120 min) -register_amd_ci(est_time=7200, suite="nightly-amd-vlm", nightly=True) +register_amd_ci(est_time=7200, suite="nightly-amd-accuracy-2-gpu-vlm", nightly=True) # AMD-verified VLM models with conservative thresholds on 100 MMMU samples # Format: (model_path, tp_size, accuracy_threshold, extra_args) diff --git a/test/registered/amd/nightly/test_gsm8k_completion_eval_amd.py b/test/registered/amd/nightly/test_gsm8k_completion_eval_amd.py deleted file mode 100644 index b44eb04ef..000000000 --- a/test/registered/amd/nightly/test_gsm8k_completion_eval_amd.py +++ /dev/null @@ -1,937 +0,0 @@ -""" -AMD GSM8K Completion Evaluation Test (Migrated from test/srt/nightly/) - -This test uses the completion-based gsm8k benchmark (few-shot prompting) -which works with base models that don't have chat templates. - -This complements test_gsm8k_eval_amd.py which uses mgsm_en (chat completions) -for instruction-tuned models. - -Base models tested here: -- GPT-OSS series (lmsys/gpt-oss-20b-bf16, lmsys/gpt-oss-120b-bf16) -- GROK series (lmzheng/grok-1, amd/grok-1-W4A8KV8, xai-org/grok-2) -- DeepSeek series (deepseek-ai/DeepSeek-V3-0324, deepseek-ai/DeepSeek-R1-0528) - -Model groups are selected via AMD_TEST_MODEL_GROUP environment variable: -- "gpt-oss" (default): GPT-OSS models only (nightly-amd-8-gpu-gpt-oss) -- "grok": All GROK models (nightly-amd-8-gpu-grok) -- "deepseek-v3-dp": DeepSeek-V3 with DP attention (nightly-amd-8-gpu-deepseek-v3-dp) -- "deepseek-v3-tc": DeepSeek-V3 with torch compile (nightly-amd-8-gpu-deepseek-v3-tc) -- "deepseek-v3-mtp": DeepSeek-V3 with MTP/EAGLE (nightly-amd-8-gpu-deepseek-v3-mtp) -- "deepseek-r1": DeepSeek-R1 reasoning model (nightly-amd-8-gpu-deepseek-r1) -- "all": All models - -Registry: nightly-amd-8-gpu suite (8-GPU tests) -""" - -import ast -import os -import re -import subprocess -import time -import unittest -from dataclasses import dataclass -from typing import List, Optional, Tuple - -import numpy as np - -# HuggingFace Hub for model cache checking and download progress -try: - from huggingface_hub import HfFileSystem, snapshot_download - from huggingface_hub.utils import GatedRepoError, RepositoryNotFoundError - - HF_HUB_AVAILABLE = True -except ImportError: - HF_HUB_AVAILABLE = False - print("[WARNING] huggingface_hub not available - model cache checking disabled") - -from sglang.srt.utils import kill_process_tree -from sglang.test.ci.ci_register import register_amd_ci -from sglang.test.test_utils import ( - DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - DEFAULT_URL_FOR_TEST, - is_in_ci, - popen_launch_server, - write_github_step_summary, -) -from sglang.utils import download_and_cache_file, read_jsonl - -# Register for AMD CI - GSM8K completion tests (~120 min) -register_amd_ci(est_time=7200, suite="nightly-amd-8-gpu", nightly=True) - -INVALID = -9999999 - - -@dataclass -class BaseModelConfig: - """Configuration for a base model to test.""" - - model_path: str - tp_size: int = 8 - accuracy_threshold: float = 0.50 - other_args: Optional[List[str]] = None - env_vars: Optional[dict] = None - tokenizer_path: Optional[str] = None - timeout: Optional[int] = None # Custom timeout for server launch (seconds) - variant: Optional[str] = ( - None # Test variant name (e.g., "basic", "MTP", "DP", "TC") - ) - - def __post_init__(self): - if self.other_args is None: - self.other_args = [] - if self.env_vars is None: - self.env_vars = {} - - def get_display_name(self) -> str: - """Return display name for logs/summary (model + variant if set).""" - if self.variant: - return f"{self.model_path} ({self.variant})" - return self.model_path - - -# ============================================================================= -# MODEL GROUPS - Each group runs on a separate 8-GPU runner -# ============================================================================= - -# Group 1: GPT-OSS models (cached on upstream CI) -# Runner: nightly-amd-8-gpu -AMD_GPT_OSS_MODELS = [ - # GPT-OSS-20B - smaller model, run first for faster feedback - BaseModelConfig( - model_path="lmsys/gpt-oss-20b-bf16", - tp_size=8, - accuracy_threshold=0.47, - other_args=[ - "--chunked-prefill-size", - "130172", - "--max-running-requests", - "128", - "--mem-fraction-static", - "0.85", - "--attention-backend", - "triton", - "--trust-remote-code", - ], - env_vars={"SGLANG_USE_AITER": "0"}, - ), - # GPT-OSS-120B - large model, needs longer timeout - BaseModelConfig( - model_path="lmsys/gpt-oss-120b-bf16", - tp_size=8, - accuracy_threshold=0.79, - timeout=900, # 15 minutes for 120B model - other_args=[ - "--chunked-prefill-size", - "130172", - "--max-running-requests", - "128", - "--mem-fraction-static", - "0.85", - "--attention-backend", - "triton", - "--trust-remote-code", - ], - env_vars={"SGLANG_USE_AITER": "0"}, - ), -] - -# Group 2: All GROK models -# Runner: nightly-amd-8-gpu-grok -# Order: GROK1-FP8 -> GROK1-IN4 -> GROK2.5 -AMD_GROK_MODELS = [ - # GROK1-FP8 - verified accuracy: 0.860, runtime: ~12.5min - BaseModelConfig( - model_path="lmzheng/grok-1", - tp_size=8, - accuracy_threshold=0.80, - timeout=3600, # 1 hour for kernel compilation - tokenizer_path="Xenova/grok-1-tokenizer", - other_args=[ - "--quantization", - "fp8", - "--attention-backend", - "aiter", - "--mem-fraction-static", - "0.85", - "--trust-remote-code", - ], - env_vars={ - "RCCL_MSCCL_ENABLE": "0", - "SGLANG_USE_AITER": "1", - "SGLANG_INT4_WEIGHT": "0", - }, - ), - # GROK1-IN4 - verified accuracy: 0.820, runtime: ~12.5min - BaseModelConfig( - model_path="amd/grok-1-W4A8KV8", - tp_size=8, - accuracy_threshold=0.80, - timeout=3600, # 1 hour for kernel compilation - tokenizer_path="Xenova/grok-1-tokenizer", - other_args=[ - "--quantization", - "fp8", - "--attention-backend", - "aiter", - "--mem-fraction-static", - "0.85", - "--trust-remote-code", - ], - env_vars={ - "RCCL_MSCCL_ENABLE": "0", - "SGLANG_USE_AITER": "1", - "SGLANG_INT4_WEIGHT": "1", - }, - ), - # GROK2.5 - verified accuracy: 0.945, runtime: ~14.5min - BaseModelConfig( - model_path="xai-org/grok-2", - tp_size=8, - accuracy_threshold=0.915, - timeout=3600, # 1 hour for download + kernel compilation - tokenizer_path="alvarobartt/grok-2-tokenizer", - other_args=[ - "--quantization", - "fp8", - "--attention-backend", - "aiter", - "--mem-fraction-static", - "0.85", - "--trust-remote-code", - ], - env_vars={ - "RCCL_MSCCL_ENABLE": "0", - "SGLANG_USE_AITER": "1", - "SGLANG_INT4_WEIGHT": "0", - }, - ), -] - -# Note: DeepSeek-V3 accuracy tests removed - V3 only used for perf tests -# See test_deepseek_v3_perf.py and test_deepseek_v31_perf.py for V3 perf tests - -# Group 3: DeepSeek-R1 (reasoning model) - Basic + MTP combined -# Runner: nightly-amd-8-gpu-deepseek-r1 -AMD_DEEPSEEK_R1_MODELS = [ - # DeepSeek-R1-0528 basic - reasoning model, ~80GB per GPU - BaseModelConfig( - model_path="deepseek-ai/DeepSeek-R1-0528", - tp_size=8, - accuracy_threshold=0.93, - timeout=3600, # 1 hour for large model - variant="basic", - other_args=[ - "--attention-backend", - "aiter", - "--chunked-prefill-size", - "131072", - "--disable-radix-cache", - "--mem-fraction-static", - "0.85", - "--trust-remote-code", - ], - env_vars={ - "SGLANG_USE_AITER": "1", - }, - ), - # DeepSeek-R1-0528 with MTP (EAGLE speculative decoding) - BaseModelConfig( - model_path="deepseek-ai/DeepSeek-R1-0528", - tp_size=8, - accuracy_threshold=0.93, - timeout=3600, - variant="MTP", - other_args=[ - "--chunked-prefill-size", - "131072", - "--speculative-algorithm", - "EAGLE", - "--speculative-num-steps", - "3", - "--speculative-eagle-topk", - "1", - "--speculative-num-draft-tokens", - "4", - "--mem-fraction-static", - "0.7", - "--trust-remote-code", - ], - env_vars={ - "SGLANG_USE_AITER": "1", - }, - ), -] - -# Group 5: DeepSeek-R1 with DP + TC combined -# Runner: nightly-amd-8-gpu-deepseek-r1-dp-tc -# Combines DP attention and Torch Compile tests for DeepSeek-R1 -AMD_DEEPSEEK_R1_DP_TC_MODELS = [ - # DeepSeek-R1-0528 with DP attention - BaseModelConfig( - model_path="deepseek-ai/DeepSeek-R1-0528", - tp_size=8, - accuracy_threshold=0.93, - timeout=3600, - variant="DP", - other_args=[ - "--chunked-prefill-size", - "131072", - "--dp-size", - "8", - "--enable-dp-attention", - "--mem-fraction-static", - "0.85", - "--trust-remote-code", - ], - env_vars={ - "SGLANG_USE_ROCM700A": "1", - "SGLANG_USE_AITER": "1", - }, - ), - # DeepSeek-R1-0528 with torch compile - BaseModelConfig( - model_path="deepseek-ai/DeepSeek-R1-0528", - tp_size=8, - accuracy_threshold=0.93, - timeout=7200, # 2 hours for compilation - variant="TC", - other_args=[ - "--chunked-prefill-size", - "131072", - "--mem-fraction-static", - "0.70", - "--cuda-graph-max-bs", - "8", - "--enable-torch-compile", - "--disable-cuda-graph", - "--trust-remote-code", - ], - env_vars={ - "SGLANG_USE_ROCM700A": "1", - "SGLANG_USE_AITER": "1", - }, - ), -] - - -def get_model_group() -> str: - """Get the model group to test from environment variable.""" - return os.environ.get("AMD_TEST_MODEL_GROUP", "gpt-oss") - - -def get_models_for_group(group: str) -> List[BaseModelConfig]: - """Get the list of models for a given group. - - Note: DeepSeek-V3 is only used for perf tests, not accuracy tests. - See test_deepseek_v3_perf.py and test_deepseek_v31_perf.py. - """ - if group == "gpt-oss": - return AMD_GPT_OSS_MODELS - elif group == "grok": - return AMD_GROK_MODELS - elif group == "deepseek-r1": - return AMD_DEEPSEEK_R1_MODELS - elif group == "deepseek-r1-dp-tc": - return AMD_DEEPSEEK_R1_DP_TC_MODELS - elif group == "deepseek-r1-all": - # All DeepSeek-R1 variants: basic, MTP, DP, TC - return AMD_DEEPSEEK_R1_MODELS + AMD_DEEPSEEK_R1_DP_TC_MODELS - elif group == "all": - return ( - AMD_GPT_OSS_MODELS - + AMD_GROK_MODELS - + AMD_DEEPSEEK_R1_MODELS - + AMD_DEEPSEEK_R1_DP_TC_MODELS - ) - else: - print(f"[WARNING] Unknown model group '{group}', using 'gpt-oss'") - return AMD_GPT_OSS_MODELS - - -# ============================================================================= -# MODEL CACHE AND DOWNLOAD UTILITIES -# ============================================================================= - - -def check_local_cache(model_path: str) -> Tuple[bool, str]: - """ - Check if model is cached locally. - - Returns: - Tuple of (is_cached, cache_path_or_message) - """ - # Check common HF cache locations - cache_dirs = [ - os.path.expanduser("~/.cache/huggingface/hub"), - "/sgl-data/hf-cache/hub", - "/home/runner/sgl-data/hf-cache", - ] - - # Convert model_path to cache directory format (org--model) - cache_name = f"models--{model_path.replace('/', '--')}" - - for cache_dir in cache_dirs: - cache_path = os.path.join(cache_dir, cache_name) - if os.path.exists(cache_path): - # Check if there are snapshots - snapshots_dir = os.path.join(cache_path, "snapshots") - if os.path.exists(snapshots_dir) and os.listdir(snapshots_dir): - return True, cache_path - - return False, f"Not found in: {', '.join(cache_dirs)}" - - -def check_hf_repo_access(model_path: str) -> Tuple[bool, str]: - """ - Check if HuggingFace repository is accessible. - - Returns: - Tuple of (is_accessible, message) - """ - if not HF_HUB_AVAILABLE: - return True, "huggingface_hub not available, skipping access check" - - try: - fs = HfFileSystem() - # Try to list files in the repo - files = fs.ls(model_path, detail=False) - if files: - return True, f"Repository accessible ({len(files)} files)" - else: - return False, "Repository exists but is empty" - except GatedRepoError: - return False, "GATED REPO - requires authentication/approval" - except RepositoryNotFoundError: - return False, "REPO NOT FOUND on HuggingFace" - except Exception as e: - error_msg = str(e) - if "401" in error_msg or "unauthorized" in error_msg.lower(): - return False, f"AUTH ERROR - may need HF_TOKEN: {error_msg[:100]}" - elif "404" in error_msg: - return False, f"NOT FOUND: {error_msg[:100]}" - elif "timeout" in error_msg.lower() or "connection" in error_msg.lower(): - return False, f"NETWORK ERROR: {error_msg[:100]}" - else: - return False, f"ERROR: {error_msg[:100]}" - - -def log_model_status(config: BaseModelConfig) -> Tuple[bool, str]: - """ - Log detailed model availability status. - - Returns: - Tuple of (is_available, status_message) - """ - model_path = config.model_path - print(f"\n📦 Checking model: {model_path}") - print("-" * 50) - - # Check local cache first - is_cached, cache_msg = check_local_cache(model_path) - if is_cached: - print(f" ✅ LOCAL CACHE: Found at {cache_msg}") - return True, f"Cached locally at {cache_msg}" - else: - print(f" ⚠️ LOCAL CACHE: {cache_msg}") - - # Check HF repo access - is_accessible, access_msg = check_hf_repo_access(model_path) - if is_accessible: - print(f" ✅ HF ACCESS: {access_msg}") - print(f" 📥 Model will be downloaded from HuggingFace (this may take a while)") - return True, f"Will download from HF: {access_msg}" - else: - print(f" ❌ HF ACCESS: {access_msg}") - return False, access_msg - - # Also check tokenizer if specified - if config.tokenizer_path: - tok_cached, tok_msg = check_local_cache(config.tokenizer_path) - if tok_cached: - print(f" ✅ TOKENIZER CACHE: Found at {tok_msg}") - else: - tok_accessible, tok_access_msg = check_hf_repo_access(config.tokenizer_path) - if tok_accessible: - print(f" ✅ TOKENIZER HF: {tok_access_msg}") - else: - print(f" ⚠️ TOKENIZER: {tok_access_msg}") - - return is_accessible, access_msg - - -def download_model_with_progress( - model_path: str, timeout: int = 3600 -) -> Tuple[bool, str]: - """ - Download model with progress logging. - - Returns: - Tuple of (success, message) - """ - if not HF_HUB_AVAILABLE: - return True, "huggingface_hub not available, skipping pre-download" - - print(f"\n📥 Pre-downloading model: {model_path}") - print(f" Timeout: {timeout}s ({timeout/60:.0f} minutes)") - print("-" * 50) - - start_time = time.time() - - try: - # Use snapshot_download which shows progress - local_dir = snapshot_download( - repo_id=model_path, - local_files_only=False, - resume_download=True, - ) - elapsed = time.time() - start_time - print(f" ✅ Download complete in {elapsed:.1f}s") - print(f" 📁 Location: {local_dir}") - return True, f"Downloaded to {local_dir}" - - except GatedRepoError: - return False, "GATED REPO - requires authentication/approval" - except RepositoryNotFoundError: - return False, "REPO NOT FOUND on HuggingFace" - except Exception as e: - error_msg = str(e) - elapsed = time.time() - start_time - if elapsed >= timeout: - return False, f"TIMEOUT after {elapsed:.0f}s: {error_msg[:100]}" - elif "timeout" in error_msg.lower() or "connection" in error_msg.lower(): - return False, f"NETWORK ERROR after {elapsed:.0f}s: {error_msg[:100]}" - else: - return False, f"ERROR after {elapsed:.0f}s: {error_msg[:100]}" - - -# ============================================================================= -# BENCHMARK UTILITIES -# ============================================================================= - - -def get_one_example(lines, i, include_answer): - """Format a single GSM8K example.""" - ret = "Question: " + lines[i]["question"] + "\nAnswer:" - if include_answer: - ret += " " + lines[i]["answer"] - return ret - - -def get_few_shot_examples(lines, k): - """Get k few-shot examples for prompting.""" - ret = "" - for i in range(k): - ret += get_one_example(lines, i, True) + "\n\n" - return ret - - -def get_answer_value(answer_str): - """Extract numerical answer from response.""" - answer_str = answer_str.replace(",", "") - numbers = re.findall(r"\d+", answer_str) - if len(numbers) < 1: - return INVALID - try: - return ast.literal_eval(numbers[-1]) - except SyntaxError: - return INVALID - - -def run_gsm8k_benchmark( - base_url: str, - num_questions: int = 200, - num_shots: int = 5, - parallel: int = 64, -) -> Tuple[float, float, float]: - """ - Run GSM8K few-shot completion benchmark. - - Returns: - Tuple of (accuracy, invalid_rate, latency) - """ - import sglang as sgl - from sglang.lang.backend.runtime_endpoint import RuntimeEndpoint - - # Download and load data - url = "https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/test.jsonl" - data_path = download_and_cache_file(url) - lines = list(read_jsonl(data_path)) - - # Construct prompts - few_shot_examples = get_few_shot_examples(lines, num_shots) - - questions = [] - labels = [] - for i in range(len(lines[:num_questions])): - questions.append(get_one_example(lines, i, False)) - labels.append(get_answer_value(lines[i]["answer"])) - assert all(l != INVALID for l in labels) - arguments = [{"question": q} for q in questions] - - # Define sglang function - @sgl.function - def few_shot_gsm8k(s, question): - s += few_shot_examples + question - s += sgl.gen( - "answer", max_tokens=512, stop=["Question", "Assistant:", "<|separator|>"] - ) - - # Set backend - backend = RuntimeEndpoint(base_url) - sgl.set_default_backend(backend) - - # Run benchmark - tic = time.perf_counter() - states = few_shot_gsm8k.run_batch( - arguments, - temperature=0, - num_threads=parallel, - progress_bar=True, - ) - latency = time.perf_counter() - tic - - # Extract predictions - preds = [] - for i in range(len(states)): - preds.append(get_answer_value(states[i]["answer"])) - - # Compute metrics - acc = np.mean(np.array(preds) == np.array(labels)) - invalid = np.mean(np.array(preds) == INVALID) - - return float(acc), float(invalid), float(latency) - - -def popen_launch_server_for_base_model( - base_url: str, - config: BaseModelConfig, -) -> "subprocess.Popen": - """Launch server for a base model with appropriate configuration.""" - # Build environment - start with current env and add config-specific vars - env = os.environ.copy() - for key, value in config.env_vars.items(): - env[key] = value - print(f"Setting env: {key}={value}") - - # Build other_args - other_args = list(config.other_args) - other_args.extend(["--tp", str(config.tp_size)]) - other_args.extend(["--log-level-http", "warning"]) - - if config.tokenizer_path: - other_args.extend(["--tokenizer-path", config.tokenizer_path]) - - # Use custom timeout if provided, otherwise use default - timeout = config.timeout if config.timeout else DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH - - process = popen_launch_server( - model=config.model_path, - base_url=base_url, - timeout=timeout, - other_args=other_args, - env=env, # Pass environment explicitly - ) - return process - - -class TestNightlyGsm8kCompletionEvalAMD(unittest.TestCase): - """ - AMD GSM8K Completion Evaluation Test - - Tests base models using few-shot completion benchmark. - This is different from mgsm_en which uses chat completions. - - Model group is selected via AMD_TEST_MODEL_GROUP env var: - - "gpt-oss": GPT-OSS models only (default, nightly-amd-8-gpu) - - "grok": All GROK models (nightly-amd-8-gpu-grok) - - "all": All models - """ - - @classmethod - def setUpClass(cls): - # Get model group from environment - cls.model_group = get_model_group() - cls.models = get_models_for_group(cls.model_group) - cls.base_url = DEFAULT_URL_FOR_TEST - cls.num_questions = int(os.environ.get("GSM8K_NUM_QUESTIONS", "200")) - - print(f"\n{'='*60}") - print(f"AMD GSM8K Completion Evaluation Test") - print(f"{'='*60}") - print(f"Model group: {cls.model_group}") - print(f"Models to test: {len(cls.models)}") - for m in cls.models: - print(f" - {m.model_path}") - print(f"Questions per model: {cls.num_questions}") - print(f"{'='*60}\n") - - def test_gsm8k_completion_all_models(self): - """Test all configured base models with GSM8K completion benchmark.""" - all_results = [] - total_test_start = time.time() - - # Summary table with runtime columns - summary = f"### Model Group: {self.model_group}\n\n" - summary += ( - "| Model | TP | Accuracy | Threshold | Startup | Bench | Total | Status |\n" - ) - summary += ( - "| ----- | -- | -------- | --------- | ------- | ----- | ----- | ------ |\n" - ) - - for config in self.models: - display_name = config.get_display_name() - with self.subTest(model=display_name): - print(f"\n{'='*60}") - print(f"Testing: {display_name} (TP={config.tp_size})") - print(f"{'='*60}") - - error_message = None - acc, invalid, latency = None, None, None - startup_time, bench_time, total_time = None, None, None - skipped = False - model_start = time.time() - - # Check model availability with detailed logging - is_available, status_msg = log_model_status(config) - - if not is_available: - print(f"\n❌ MODEL NOT AVAILABLE: {status_msg}") - print(f"⏭️ SKIPPING: {display_name}") - status = f"⏭️ SKIP" - skipped = True - all_results.append( - { - "model": display_name, - "tp_size": config.tp_size, - "accuracy": None, - "threshold": config.accuracy_threshold, - "invalid": None, - "latency": None, - "startup_time": None, - "bench_time": None, - "total_time": None, - "passed": True, # Don't count as failure - "skipped": True, - "error": status_msg, - } - ) - else: - try: - # Launch server with timing - print(f"\n🚀 Launching server for {display_name}...") - server_start = time.time() - process = popen_launch_server_for_base_model( - self.base_url, config - ) - startup_time = time.time() - server_start - print(f"⏱️ Server startup: {startup_time:.1f}s") - - try: - # Run benchmark with timing and retries - print( - f"📊 Running GSM8K benchmark ({self.num_questions} questions)..." - ) - bench_start = time.time() - acc, invalid, latency = None, None, None - for attempt in range(3): - try: - acc, invalid, latency = run_gsm8k_benchmark( - self.base_url, - num_questions=self.num_questions, - num_shots=5, - parallel=64, - ) - print( - f" Attempt {attempt + 1}: accuracy={acc:.3f}" - ) - if acc >= config.accuracy_threshold: - break - except Exception as e: - print( - f" Attempt {attempt + 1} failed with error: {e}" - ) - if attempt == 2: - raise - bench_time = time.time() - bench_start - - total_time = time.time() - model_start - - print(f"\n📈 Results for {display_name}:") - print( - f" Accuracy: {acc:.3f} (threshold: {config.accuracy_threshold})" - ) - print(f" Invalid: {invalid:.3f}") - print(f" Benchmark latency: {latency:.1f}s") - print(f"\n⏱️ Runtime breakdown:") - print(f" Server startup: {startup_time:.1f}s") - print(f" Benchmark: {bench_time:.1f}s") - print(f" Total: {total_time:.1f}s") - - passed = acc >= config.accuracy_threshold - status = "✅ PASS" if passed else "❌ FAIL" - - if passed: - print(f"\n Status: ✅ PASSED") - else: - print(f"\n Status: ❌ FAILED (below threshold)") - - all_results.append( - { - "model": display_name, - "tp_size": config.tp_size, - "accuracy": acc, - "threshold": config.accuracy_threshold, - "invalid": invalid, - "latency": latency, - "startup_time": startup_time, - "bench_time": bench_time, - "total_time": total_time, - "passed": passed, - "skipped": False, - "error": None, - } - ) - - except Exception as e: - error_message = str(e) - total_time = time.time() - model_start - print(f"\n❌ Error during benchmark: {error_message}") - status = "❌ ERROR" - all_results.append( - { - "model": display_name, - "tp_size": config.tp_size, - "accuracy": None, - "threshold": config.accuracy_threshold, - "invalid": None, - "latency": None, - "startup_time": startup_time, - "bench_time": None, - "total_time": total_time, - "passed": False, - "skipped": False, - "error": error_message, - } - ) - - finally: - print(f"\n🛑 Stopping server for {display_name}...") - kill_process_tree(process.pid) - - except Exception as e: - error_message = str(e) - total_time = time.time() - model_start - print(f"\n❌ Error launching server: {error_message}") - status = "❌ ERROR" - all_results.append( - { - "model": display_name, - "tp_size": config.tp_size, - "accuracy": None, - "threshold": config.accuracy_threshold, - "invalid": None, - "latency": None, - "startup_time": None, - "bench_time": None, - "total_time": total_time, - "passed": False, - "skipped": False, - "error": error_message, - } - ) - - # Add to summary with runtime (use display name to show variant) - acc_str = f"{acc:.3f}" if acc is not None else "N/A" - startup_str = ( - f"{startup_time:.0f}s" if startup_time is not None else "N/A" - ) - bench_str = f"{bench_time:.0f}s" if bench_time is not None else "N/A" - total_str = f"{total_time:.0f}s" if total_time is not None else "N/A" - summary += f"| {display_name} | {config.tp_size} | {acc_str} | {config.accuracy_threshold} | {startup_str} | {bench_str} | {total_str} | {status} |\n" - - # Calculate total test runtime - total_test_time = time.time() - total_test_start - - # Print summary - print(f"\n{'='*60}") - print(f"SUMMARY - Model Group: {self.model_group}") - print(f"{'='*60}") - print(summary) - print( - f"\n⏱️ Total test runtime: {total_test_time:.1f}s ({total_test_time/60:.1f} min)" - ) - - # Check for failures (exclude skipped models) - failed_models = [ - r for r in all_results if not r["passed"] and not r.get("skipped", False) - ] - skipped_models = [r for r in all_results if r.get("skipped", False)] - passed_models = [ - r for r in all_results if r["passed"] and not r.get("skipped", False) - ] - - # Build GitHub summary with results and failure details - # Note: summary already includes the "### Model Group:" header - github_summary = f"{summary}\n" - github_summary += f"\n**Statistics:** ✅ Passed: {len(passed_models)} | ❌ Failed: {len(failed_models)} | ⏭️ Skipped: {len(skipped_models)}\n" - github_summary += f"\n**Total Runtime:** {total_test_time:.1f}s ({total_test_time/60:.1f} min)\n" - - if failed_models: - github_summary += "\n#### ❌ Failed Models\n" - for r in failed_models: - acc_str = f"{r['accuracy']:.3f}" if r["accuracy"] is not None else "N/A" - github_summary += f"- **{r['model']}**: accuracy={acc_str}, threshold={r['threshold']}" - if r.get("error"): - # Truncate long errors for display - error_short = ( - r["error"][:200] + "..." - if len(r["error"]) > 200 - else r["error"] - ) - github_summary += f"\n - Error: `{error_short}`" - github_summary += "\n" - - if skipped_models: - github_summary += "\n#### ⏭️ Skipped Models\n" - for r in skipped_models: - github_summary += ( - f"- **{r['model']}**: {r.get('error', 'Not available')}\n" - ) - - # Write GitHub step summary - if is_in_ci(): - write_github_step_summary(github_summary) - - print(f"\n📊 Final Statistics:") - print(f" Passed: {len(passed_models)}") - print(f" Failed: {len(failed_models)}") - print(f" Skipped: {len(skipped_models)}") - - if skipped_models: - print(f"\n⏭️ Skipped models (not available):") - for r in skipped_models: - print(f" - {r['model']}: {r['error']}") - - if failed_models: - print(f"\n❌ Failed models:") - for r in failed_models: - acc_str = f"{r['accuracy']:.3f}" if r["accuracy"] is not None else "N/A" - print( - f" - {r['model']}: accuracy={acc_str}, threshold={r['threshold']}" - ) - if r.get("error"): - print(f" Error: {r['error'][:200]}") - - failure_msg = "\n".join( - [ - f"- {r['model']}: accuracy={r['accuracy']}, threshold={r['threshold']}, error={r['error']}" - for r in failed_models - ] - ) - raise AssertionError(f"The following models failed:\n{failure_msg}") - - -if __name__ == "__main__": - unittest.main() diff --git a/test/registered/amd/nightly/test_gsm8k_completion_eval_mi35x.py b/test/registered/amd/nightly/test_gsm8k_completion_eval_mi35x.py deleted file mode 100644 index 87a95c023..000000000 --- a/test/registered/amd/nightly/test_gsm8k_completion_eval_mi35x.py +++ /dev/null @@ -1,726 +0,0 @@ -""" -MI35x GSM8K Completion Evaluation Test (8-GPU) - -This test uses the completion-based gsm8k benchmark (few-shot prompting) -for MI35x-specific models that differ from MI300X configurations. - -MI35x-specific models: -- GPT-OSS series: Uses openai/gpt-oss-* (not lmsys/gpt-oss-*-bf16) -- DeepSeek-R1-0528: Same model as MI300X (MXFP4 only used for perf tests) - -Model groups are selected via AMD_TEST_MODEL_GROUP environment variable: -- "gpt-oss" (default): GPT-OSS models with MI35x paths -- "deepseek-r1": DeepSeek-R1-0528 basic + MTP (same as MI300X) -- "deepseek-r1-dp-tc": DeepSeek-R1-0528 DP + TC (same as MI300X) -- "deepseek-r1-all": All DeepSeek-R1-0528 variants (basic, MTP, DP, TC) - -Registry: nightly-amd-8-gpu-mi35x suite (8-GPU tests on MI35x) -""" - -import ast -import os - -# Set HF cache to /data2/models/ for MI35x so HF models download there -os.environ.setdefault("HF_HOME", "/data2/models/huggingface") -os.environ.setdefault("HF_HUB_CACHE", "/data2/models/huggingface/hub") -import re -import subprocess -import time -import unittest -from dataclasses import dataclass -from typing import List, Optional, Tuple - -import numpy as np - -# HuggingFace Hub for model cache checking and download progress -try: - from huggingface_hub import HfFileSystem - from huggingface_hub.utils import GatedRepoError, RepositoryNotFoundError - - HF_HUB_AVAILABLE = True -except ImportError: - HF_HUB_AVAILABLE = False - print("[WARNING] huggingface_hub not available - model cache checking disabled") - -from sglang.srt.utils import kill_process_tree -from sglang.test.ci.ci_register import register_amd_ci -from sglang.test.test_utils import ( - DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - DEFAULT_URL_FOR_TEST, - is_in_ci, - popen_launch_server, - write_github_step_summary, -) -from sglang.utils import download_and_cache_file, read_jsonl - -# Register for AMD CI - MI35x 8-GPU GSM8K completion tests (~120 min) -register_amd_ci(est_time=7200, suite="nightly-amd-8-gpu-mi35x", nightly=True) - -INVALID = -9999999 - - -@dataclass -class BaseModelConfig: - """Configuration for a base model to test.""" - - model_path: str # HuggingFace model ID (e.g., "amd/DeepSeek-R1-MXFP4-Preview") - tp_size: int = 8 - accuracy_threshold: float = 0.50 - other_args: Optional[List[str]] = None - env_vars: Optional[dict] = None - tokenizer_path: Optional[str] = None - timeout: Optional[int] = None - local_path: Optional[str] = None # Preferred local path (checked first before HF) - variant: Optional[str] = ( - None # Test variant name (e.g., "basic", "MTP", "DP", "TC") - ) - - def __post_init__(self): - if self.other_args is None: - self.other_args = [] - if self.env_vars is None: - self.env_vars = {} - - def get_effective_model_path(self) -> str: - """Return local_path if it exists, otherwise model_path (HF ID).""" - if self.local_path and os.path.exists(self.local_path): - return self.local_path - return self.model_path - - def get_display_name(self) -> str: - """Return display name for logs/summary (model + variant if set).""" - if self.variant: - return f"{self.model_path} ({self.variant})" - return self.model_path - - -# ============================================================================= -# MI35x MODEL GROUPS - Different from MI300X configurations -# ============================================================================= - -# Group 1: GPT-OSS models (MI35x uses openai/* paths, not lmsys/*) -MI35X_GPT_OSS_MODELS = [ - # GPT-OSS-20B - MI35x specific path - BaseModelConfig( - model_path="openai/gpt-oss-20b", - tp_size=8, - accuracy_threshold=0.47, - other_args=[ - "--chunked-prefill-size", - "130172", - "--max-running-requests", - "128", - "--mem-fraction-static", - "0.85", - "--attention-backend", - "triton", - "--trust-remote-code", - ], - env_vars={"SGLANG_USE_AITER": "1"}, - ), - # GPT-OSS-120B - MI35x specific path - BaseModelConfig( - model_path="openai/gpt-oss-120b", - tp_size=8, - accuracy_threshold=0.79, - timeout=900, # 15 minutes for 120B model - other_args=[ - "--chunked-prefill-size", - "130172", - "--max-running-requests", - "128", - "--mem-fraction-static", - "0.85", - "--attention-backend", - "triton", - "--trust-remote-code", - ], - env_vars={"SGLANG_USE_AITER": "1"}, - ), -] - -# Group 2: DeepSeek-R1-0528 basic + MTP (same model as MI300X for consistency) -# Runner: nightly-test-8-gpu-mi35x-deepseek-r1 -# Note: MXFP4 variant only used for perf tests (test_deepseek_r1_mxfp4_perf.py) -MI35X_DEEPSEEK_R1_MODELS = [ - # DeepSeek-R1-0528 basic - reasoning model, ~80GB per GPU - BaseModelConfig( - model_path="deepseek-ai/DeepSeek-R1-0528", - tp_size=8, - accuracy_threshold=0.93, - timeout=3600, # 1 hour for large model - variant="basic", - other_args=[ - "--attention-backend", - "aiter", - "--chunked-prefill-size", - "131072", - "--disable-radix-cache", - "--mem-fraction-static", - "0.85", - "--trust-remote-code", - ], - env_vars={ - "SGLANG_USE_AITER": "1", - }, - ), - # DeepSeek-R1-0528 with MTP (EAGLE speculative decoding) - BaseModelConfig( - model_path="deepseek-ai/DeepSeek-R1-0528", - tp_size=8, - accuracy_threshold=0.93, - timeout=3600, - variant="MTP", - other_args=[ - "--chunked-prefill-size", - "131072", - "--speculative-algorithm", - "EAGLE", - "--speculative-num-steps", - "3", - "--speculative-eagle-topk", - "1", - "--speculative-num-draft-tokens", - "4", - "--mem-fraction-static", - "0.7", - "--trust-remote-code", - ], - env_vars={ - "SGLANG_USE_AITER": "1", - }, - ), -] - -# Group 3: DeepSeek-R1-0528 with DP + TC (requires ROCm 7.0+) -# Runner: nightly-test-8-gpu-mi35x-deepseek-r1-dp-tc -MI35X_DEEPSEEK_R1_DP_TC_MODELS = [ - # DeepSeek-R1-0528 with DP attention - BaseModelConfig( - model_path="deepseek-ai/DeepSeek-R1-0528", - tp_size=8, - accuracy_threshold=0.93, - timeout=3600, - variant="DP", - other_args=[ - "--chunked-prefill-size", - "131072", - "--dp-size", - "8", - "--enable-dp-attention", - "--mem-fraction-static", - "0.85", - "--trust-remote-code", - ], - env_vars={ - "SGLANG_USE_ROCM700A": "1", - "SGLANG_USE_AITER": "1", - }, - ), - # DeepSeek-R1-0528 with torch compile - BaseModelConfig( - model_path="deepseek-ai/DeepSeek-R1-0528", - tp_size=8, - accuracy_threshold=0.93, - timeout=7200, # 2 hours for compilation - variant="TC", - other_args=[ - "--chunked-prefill-size", - "131072", - "--mem-fraction-static", - "0.70", - "--cuda-graph-max-bs", - "8", - "--enable-torch-compile", - "--disable-cuda-graph", - "--trust-remote-code", - ], - env_vars={ - "SGLANG_USE_ROCM700A": "1", - "SGLANG_USE_AITER": "1", - }, - ), -] - - -def get_model_group() -> str: - """Get the model group to test from environment variable.""" - return os.environ.get("AMD_TEST_MODEL_GROUP", "gpt-oss") - - -def get_models_for_group(group: str) -> List[BaseModelConfig]: - """Get the list of models for a given group. - - Note: DeepSeek-R1-MXFP4 is only used for perf tests, not accuracy tests. - See test_deepseek_r1_mxfp4_perf.py for MXFP4 perf tests. - """ - if group == "gpt-oss": - return MI35X_GPT_OSS_MODELS - elif group == "deepseek-r1": - return MI35X_DEEPSEEK_R1_MODELS - elif group == "deepseek-r1-dp-tc": - return MI35X_DEEPSEEK_R1_DP_TC_MODELS - elif group == "deepseek-r1-all": - # All DeepSeek-R1-0528 variants: basic, MTP, DP, TC - return MI35X_DEEPSEEK_R1_MODELS + MI35X_DEEPSEEK_R1_DP_TC_MODELS - elif group == "all": - return ( - MI35X_GPT_OSS_MODELS - + MI35X_DEEPSEEK_R1_MODELS - + MI35X_DEEPSEEK_R1_DP_TC_MODELS - ) - else: - print(f"[WARNING] Unknown model group '{group}', using 'gpt-oss'") - return MI35X_GPT_OSS_MODELS - - -# ============================================================================= -# MODEL CACHE AND DOWNLOAD UTILITIES -# ============================================================================= - - -def check_local_cache(model_path: str) -> Tuple[bool, str]: - """ - Check if model is cached locally. - - Returns: - Tuple of (is_cached, cache_path_or_message) - """ - # Check common HF cache locations for MI35x - cache_dirs = [ - os.path.expanduser("~/.cache/huggingface/hub"), - "/data2/models/huggingface/hub", - os.environ.get("HF_HUB_CACHE", ""), - ] - cache_dirs = [d for d in cache_dirs if d] # Remove empty - - # Convert model_path to cache directory format (org--model) - cache_name = f"models--{model_path.replace('/', '--')}" - - for cache_dir in cache_dirs: - cache_path = os.path.join(cache_dir, cache_name) - if os.path.exists(cache_path): - # Check if there are snapshots - snapshots_dir = os.path.join(cache_path, "snapshots") - if os.path.exists(snapshots_dir) and os.listdir(snapshots_dir): - return True, cache_path - - return False, f"Not found in: {', '.join(cache_dirs)}" - - -def check_hf_repo_access(model_path: str) -> Tuple[bool, str]: - """ - Check if HuggingFace repository is accessible. - - Returns: - Tuple of (is_accessible, message) - """ - if not HF_HUB_AVAILABLE: - return True, "huggingface_hub not available, skipping access check" - - try: - fs = HfFileSystem() - # Try to list files in the repo - files = fs.ls(model_path, detail=False) - if files: - return True, f"Repository accessible ({len(files)} files)" - else: - return False, "Repository exists but is empty" - except GatedRepoError: - return False, "GATED REPO - requires authentication/approval" - except RepositoryNotFoundError: - return False, "REPO NOT FOUND on HuggingFace" - except Exception as e: - error_msg = str(e) - if "401" in error_msg or "unauthorized" in error_msg.lower(): - return False, f"AUTH ERROR - may need HF_TOKEN: {error_msg[:100]}" - elif "404" in error_msg: - return False, f"NOT FOUND: {error_msg[:100]}" - elif "timeout" in error_msg.lower() or "connection" in error_msg.lower(): - return False, f"NETWORK ERROR: {error_msg[:100]}" - else: - return False, f"ERROR: {error_msg[:100]}" - - -def log_model_status(config: "BaseModelConfig") -> Tuple[bool, str]: - """ - Log detailed model availability status. - - Checks in order: - 1. local_path (if specified) - preferred local path - 2. model_path as local path (if starts with /) - 3. model_path as HF model ID - check cache then HF access - - Returns: - Tuple of (is_available, status_message) - """ - model_path = config.model_path - local_path = config.local_path - - print(f"\n📦 Checking model: {model_path}") - if local_path: - print(f" (preferred local: {local_path})") - print("-" * 50) - - # Step 1: Check preferred local_path first (if specified) - if local_path: - if os.path.exists(local_path): - print(f" ✅ LOCAL PATH: Found at {local_path}") - return True, f"Local path exists at {local_path}" - else: - print(f" ⚠️ LOCAL PATH: Not found at {local_path}, trying HF fallback...") - - # Step 2: For absolute paths (starting with /), check if exists - if model_path.startswith("/"): - if os.path.exists(model_path): - print(f" ✅ LOCAL PATH: Found at {model_path}") - return True, f"Local path exists at {model_path}" - else: - print(f" ❌ LOCAL PATH: Not found at {model_path}") - return False, f"Local path not found at {model_path}" - - # Step 3: For HF model IDs, check local cache first - is_cached, cache_msg = check_local_cache(model_path) - if is_cached: - print(f" ✅ LOCAL CACHE: Found at {cache_msg}") - return True, f"Cached locally at {cache_msg}" - else: - print(f" ⚠️ LOCAL CACHE: {cache_msg}") - - # Step 4: Check HF repo access (will download if accessible) - is_accessible, access_msg = check_hf_repo_access(model_path) - if is_accessible: - print(f" ✅ HF ACCESS: {access_msg}") - print( - f" 📥 Model will be downloaded from HuggingFace to {os.environ.get('HF_HOME', '~/.cache/huggingface')}" - ) - return True, f"Will download from HF: {access_msg}" - else: - print(f" ❌ HF ACCESS: {access_msg}") - return False, access_msg - - -# ============================================================================= -# BENCHMARK UTILITIES -# ============================================================================= - - -def get_one_example(lines, i, include_answer): - """Format a single GSM8K example.""" - ret = "Question: " + lines[i]["question"] + "\nAnswer:" - if include_answer: - ret += " " + lines[i]["answer"] - return ret - - -def get_few_shot_examples(lines, k): - """Get k few-shot examples for prompting.""" - ret = "" - for i in range(k): - ret += get_one_example(lines, i, True) + "\n\n" - return ret - - -def get_answer_value(answer_str): - """Extract numerical answer from response.""" - answer_str = answer_str.replace(",", "") - numbers = re.findall(r"\d+", answer_str) - if len(numbers) < 1: - return INVALID - try: - return ast.literal_eval(numbers[-1]) - except SyntaxError: - return INVALID - - -def run_gsm8k_benchmark( - base_url: str, - num_questions: int = 200, - num_shots: int = 5, - parallel: int = 64, -) -> Tuple[float, float, float]: - """Run GSM8K few-shot completion benchmark.""" - import sglang as sgl - from sglang.lang.backend.runtime_endpoint import RuntimeEndpoint - - url = "https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/test.jsonl" - data_path = download_and_cache_file(url) - lines = list(read_jsonl(data_path)) - - few_shot_examples = get_few_shot_examples(lines, num_shots) - - questions = [] - labels = [] - for i in range(len(lines[:num_questions])): - questions.append(get_one_example(lines, i, False)) - labels.append(get_answer_value(lines[i]["answer"])) - assert all(l != INVALID for l in labels) - arguments = [{"question": q} for q in questions] - - @sgl.function - def few_shot_gsm8k(s, question): - s += few_shot_examples + question - s += sgl.gen( - "answer", max_tokens=512, stop=["Question", "Assistant:", "<|separator|>"] - ) - - backend = RuntimeEndpoint(base_url) - sgl.set_default_backend(backend) - - tic = time.perf_counter() - states = few_shot_gsm8k.run_batch( - arguments, - temperature=0, - num_threads=parallel, - progress_bar=True, - ) - latency = time.perf_counter() - tic - - preds = [] - for i in range(len(states)): - preds.append(get_answer_value(states[i]["answer"])) - - acc = np.mean(np.array(preds) == np.array(labels)) - invalid = np.mean(np.array(preds) == INVALID) - - return float(acc), float(invalid), float(latency) - - -def popen_launch_server_for_base_model( - base_url: str, - config: BaseModelConfig, -) -> "subprocess.Popen": - """Launch server for a base model with appropriate configuration.""" - env = os.environ.copy() - for key, value in config.env_vars.items(): - env[key] = value - print(f"Setting env: {key}={value}") - - other_args = list(config.other_args) - other_args.extend(["--tp", str(config.tp_size)]) - other_args.extend(["--log-level-http", "warning"]) - - if config.tokenizer_path: - other_args.extend(["--tokenizer-path", config.tokenizer_path]) - - timeout = config.timeout if config.timeout else DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH - - # Use effective model path (local if exists, else HF model ID) - effective_model_path = config.get_effective_model_path() - print(f"Using model path: {effective_model_path}") - - process = popen_launch_server( - model=effective_model_path, - base_url=base_url, - timeout=timeout, - other_args=other_args, - env=env, - ) - return process - - -class TestMI35xGsm8kCompletionEval(unittest.TestCase): - """MI35x GSM8K Completion Evaluation Test (8-GPU) - - Tests MI35x-specific base models using few-shot completion benchmark. - """ - - @classmethod - def setUpClass(cls): - cls.model_group = get_model_group() - cls.models = get_models_for_group(cls.model_group) - cls.base_url = DEFAULT_URL_FOR_TEST - cls.num_questions = int(os.environ.get("GSM8K_NUM_QUESTIONS", "200")) - - print(f"\n{'='*60}") - print(f"MI35x GSM8K Completion Evaluation Test (8-GPU)") - print(f"{'='*60}") - print(f"Model group: {cls.model_group}") - print(f"Models to test: {len(cls.models)}") - for m in cls.models: - print(f" - {m.model_path}") - print(f"Questions per model: {cls.num_questions}") - print(f"{'='*60}\n") - - def test_gsm8k_completion_all_models(self): - """Test all configured MI35x models with GSM8K completion benchmark.""" - all_results = [] - total_test_start = time.time() - - summary = f"### MI35x Model Group: {self.model_group}\n\n" - summary += ( - "| Model | TP | Accuracy | Threshold | Startup | Bench | Total | Status |\n" - ) - summary += ( - "| ----- | -- | -------- | --------- | ------- | ----- | ----- | ------ |\n" - ) - - for config in self.models: - display_name = config.get_display_name() - with self.subTest(model=display_name): - print(f"\n{'='*60}") - print(f"Testing: {display_name} (TP={config.tp_size})") - print(f"{'='*60}") - - error_message = None - acc, invalid, latency = None, None, None - startup_time, bench_time, total_time = None, None, None - model_start = time.time() - - # Check model availability with detailed logging - is_available, status_msg = log_model_status(config) - - if not is_available: - print(f"\n❌ MODEL NOT AVAILABLE: {status_msg}") - print(f"⏭️ SKIPPING: {display_name}") - status = "⏭️ SKIP" - all_results.append( - { - "model": display_name, - "tp_size": config.tp_size, - "accuracy": None, - "threshold": config.accuracy_threshold, - "passed": True, - "skipped": True, - "error": status_msg, - } - ) - else: - try: - print(f"\n🚀 Launching server for {display_name}...") - server_start = time.time() - process = popen_launch_server_for_base_model( - self.base_url, config - ) - startup_time = time.time() - server_start - print(f"⏱️ Server startup: {startup_time:.1f}s") - - try: - print( - f"📊 Running GSM8K benchmark ({self.num_questions} questions)..." - ) - bench_start = time.time() - for attempt in range(3): - try: - acc, invalid, latency = run_gsm8k_benchmark( - self.base_url, - num_questions=self.num_questions, - num_shots=5, - parallel=64, - ) - print( - f" Attempt {attempt + 1}: accuracy={acc:.3f}" - ) - if acc >= config.accuracy_threshold: - break - except Exception as e: - print(f" Attempt {attempt + 1} failed: {e}") - if attempt == 2: - raise - bench_time = time.time() - bench_start - total_time = time.time() - model_start - - passed = acc >= config.accuracy_threshold - status = "✅ PASS" if passed else "❌ FAIL" - - print( - f"\n📈 Results: accuracy={acc:.3f} (threshold: {config.accuracy_threshold})" - ) - print(f"⏱️ Total: {total_time:.1f}s") - - all_results.append( - { - "model": display_name, - "tp_size": config.tp_size, - "accuracy": acc, - "threshold": config.accuracy_threshold, - "startup_time": startup_time, - "bench_time": bench_time, - "total_time": total_time, - "passed": passed, - "skipped": False, - "error": None, - } - ) - - except Exception as e: - error_message = str(e) - total_time = time.time() - model_start - print(f"\n❌ Error: {error_message}") - status = "❌ ERROR" - all_results.append( - { - "model": display_name, - "tp_size": config.tp_size, - "accuracy": None, - "threshold": config.accuracy_threshold, - "passed": False, - "skipped": False, - "error": error_message, - } - ) - - finally: - print(f"\n🛑 Stopping server...") - kill_process_tree(process.pid) - - except Exception as e: - error_message = str(e) - total_time = time.time() - model_start - print(f"\n❌ Error launching server: {error_message}") - status = "❌ ERROR" - all_results.append( - { - "model": display_name, - "tp_size": config.tp_size, - "accuracy": None, - "threshold": config.accuracy_threshold, - "passed": False, - "skipped": False, - "error": error_message, - } - ) - - # Add to summary (use display name to show variant) - acc_str = f"{acc:.3f}" if acc is not None else "N/A" - startup_str = ( - f"{startup_time:.0f}s" if startup_time is not None else "N/A" - ) - bench_str = f"{bench_time:.0f}s" if bench_time is not None else "N/A" - total_str = f"{total_time:.0f}s" if total_time is not None else "N/A" - summary += f"| {display_name} | {config.tp_size} | {acc_str} | {config.accuracy_threshold} | {startup_str} | {bench_str} | {total_str} | {status} |\n" - - # Final summary - total_test_time = time.time() - total_test_start - failed_models = [ - r for r in all_results if not r["passed"] and not r.get("skipped", False) - ] - skipped_models = [r for r in all_results if r.get("skipped", False)] - passed_models = [ - r for r in all_results if r["passed"] and not r.get("skipped", False) - ] - - print(f"\n{'='*60}") - print(f"SUMMARY - MI35x Model Group: {self.model_group}") - print(f"{'='*60}") - print(summary) - print( - f"\n📊 Passed: {len(passed_models)} | Failed: {len(failed_models)} | Skipped: {len(skipped_models)}" - ) - print(f"⏱️ Total: {total_test_time:.1f}s ({total_test_time/60:.1f} min)") - - if is_in_ci(): - write_github_step_summary(summary) - - if failed_models: - failure_msg = "\n".join( - [ - f"- {r['model']}: {r.get('error', 'below threshold')}" - for r in failed_models - ] - ) - raise AssertionError(f"The following models failed:\n{failure_msg}") - - -if __name__ == "__main__": - unittest.main() diff --git a/test/registered/amd/test_deepseek_r1_mxfp4_perf.py b/test/registered/amd/perf/mi35x/test_deepseek_r1_mxfp4_perf_mi35x.py similarity index 91% rename from test/registered/amd/test_deepseek_r1_mxfp4_perf.py rename to test/registered/amd/perf/mi35x/test_deepseek_r1_mxfp4_perf_mi35x.py index e1a0de107..7660d0b6e 100644 --- a/test/registered/amd/test_deepseek_r1_mxfp4_perf.py +++ b/test/registered/amd/perf/mi35x/test_deepseek_r1_mxfp4_perf_mi35x.py @@ -1,11 +1,13 @@ -"""Nightly performance benchmark for DeepSeek-R1-MXFP4 model (MI35x). +"""MI35x Nightly performance benchmark for DeepSeek-R1-MXFP4 model. This test benchmarks the DeepSeek-R1-MXFP4 quantized model on MI35x with 8 GPUs. The model path can be configured via DEEPSEEK_R1_MXFP4_MODEL_PATH environment variable. +Registry: nightly-perf-8-gpu-mi35x-deepseek-r1-mxfp4 suite + Example usage: - DEEPSEEK_R1_MXFP4_MODEL_PATH=/data2/models/amd-DeepSeek-R1-MXFP4-Preview python -m pytest test_deepseek_r1_mxfp4_perf.py -v + DEEPSEEK_R1_MXFP4_MODEL_PATH=/data2/models/amd-DeepSeek-R1-MXFP4-Preview python -m pytest test_deepseek_r1_mxfp4_perf_mi35x.py -v """ import os @@ -21,9 +23,9 @@ from sglang.test.nightly_bench_utils import BenchmarkResult from sglang.test.nightly_utils import NightlyBenchmarkRunner from sglang.test.test_utils import DEFAULT_URL_FOR_TEST, _parse_int_list_env -# Register for AMD CI - DeepSeek-R1-MXFP4 benchmark (~300 min) +# Register for AMD CI - DeepSeek-R1-MXFP4 benchmark on MI35x (~300 min) register_amd_ci( - est_time=18000, suite="nightly-perf-8-gpu-deepseek-r1-mxfp4", nightly=True + est_time=18000, suite="nightly-perf-8-gpu-mi35x-deepseek-r1-mxfp4", nightly=True ) @@ -33,7 +35,7 @@ def generate_simple_markdown_report(results: List[BenchmarkResult]) -> str: if results[0].run_name and results[0].run_name != "default": model_header += f" ({results[0].run_name})" - gpu_config = os.getenv("GPU_CONFIG", "") + gpu_config = os.getenv("GPU_CONFIG", "MI35x") if gpu_config: model_header += f" [{gpu_config}]" @@ -52,7 +54,7 @@ def generate_simple_markdown_report(results: List[BenchmarkResult]) -> str: # Priority: 1) env var, 2) local path, 3) HuggingFace model ID DEEPSEEK_R1_MXFP4_LOCAL_PATH = "/data2/models/amd-DeepSeek-R1-MXFP4-Preview" DEEPSEEK_R1_MXFP4_HF_MODEL_ID = "amd/DeepSeek-R1-MXFP4-Preview" -PROFILE_DIR = "performance_profiles_deepseek_r1_mxfp4" +PROFILE_DIR = "performance_profiles_deepseek_r1_mxfp4_mi35x" def get_model_path() -> str: @@ -68,8 +70,8 @@ def get_model_path() -> str: return DEEPSEEK_R1_MXFP4_HF_MODEL_ID -class TestNightlyDeepseekR1MXFP4Performance(unittest.TestCase): - """Nightly performance benchmark for DeepSeek-R1-MXFP4 model (MI35x). +class TestDeepseekR1MXFP4PerfMI35x(unittest.TestCase): + """MI35x Nightly performance benchmark for DeepSeek-R1-MXFP4 model. Tests the DeepSeek-R1-MXFP4 quantized model on TP=8 with DP=8. Uses local path if available, otherwise downloads from HuggingFace. diff --git a/test/registered/amd/perf/mi35x/test_grok1_int4_perf_mi35x.py b/test/registered/amd/perf/mi35x/test_grok1_int4_perf_mi35x.py new file mode 100644 index 000000000..5ffd8d000 --- /dev/null +++ b/test/registered/amd/perf/mi35x/test_grok1_int4_perf_mi35x.py @@ -0,0 +1,126 @@ +"""MI35x Nightly performance benchmark for Grok-1 INT4 (W4A8KV8). + +This test benchmarks Grok-1 (314B MOE) with INT4 weight quantization on 8 GPUs. + +Registry: nightly-perf-8-gpu-mi35x-grok1-int4 suite +""" + +import os +import unittest +from typing import List + +from sglang.test.ci.ci_register import register_amd_ci +from sglang.test.nightly_bench_utils import BenchmarkResult +from sglang.test.nightly_utils import NightlyBenchmarkRunner +from sglang.test.test_utils import DEFAULT_URL_FOR_TEST, _parse_int_list_env + +# Register for AMD CI - Grok-1 INT4 benchmark on MI35x (~25 min) +register_amd_ci( + est_time=1500, suite="nightly-perf-8-gpu-mi35x-grok1-int4", nightly=True +) + + +def generate_simple_markdown_report(results: List[BenchmarkResult]) -> str: + """Generate a simplified markdown report without traces and cost columns.""" + model_header = results[0].model_path + if results[0].run_name and results[0].run_name != "default": + model_header += f" ({results[0].run_name})" + + gpu_config = os.getenv("GPU_CONFIG", "MI35x") + if gpu_config: + model_header += f" [{gpu_config}]" + + summary = f"### {model_header}\n" + summary += "| batch size | input len | latency (s) | input throughput (tok/s) | output throughput (tok/s) | ITL (ms) |\n" + summary += "| ---------- | --------- | ----------- | ------------------------ | ------------------------- | -------- |\n" + + for result in results: + itl = 1 / (result.output_throughput / result.batch_size) * 1000 + summary += f"| {result.batch_size} | {result.input_len} | {result.latency:.2f} | {result.input_throughput:.2f} | {result.output_throughput:.2f} | {itl:.2f} |\n" + + return summary + + +# Model and tokenizer paths can be overridden via environment variables +GROK1_MODEL_PATH = os.environ.get("GROK1_MODEL_PATH", "amd/grok-1-W4A8KV8") +GROK1_TOKENIZER_PATH = os.environ.get("GROK1_TOKENIZER_PATH", "Xenova/grok-1-tokenizer") +PROFILE_DIR = "performance_profiles_grok1_int4_mi35x" + + +class TestGrok1INT4PerfMI35x(unittest.TestCase): + """Test suite for Grok-1 INT4 performance benchmarks on MI35x.""" + + @classmethod + def setUpClass(cls): + cls.base_url = DEFAULT_URL_FOR_TEST + cls.batch_sizes = [1, 1, 8, 16, 64] + cls.input_lens = tuple(_parse_int_list_env("NIGHTLY_INPUT_LENS", "1024")) + cls.output_lens = tuple(_parse_int_list_env("NIGHTLY_OUTPUT_LENS", "512")) + + cls.model_config = { + "name": "grok1-int4-mi35x", + "model_path": GROK1_MODEL_PATH, + "other_args": [ + "--trust-remote-code", + "--tp", + "8", + "--quantization", + "fp8", + "--mem-fraction-static", + "0.85", + "--tokenizer-path", + GROK1_TOKENIZER_PATH, + "--attention-backend", + "aiter", + ], + "env_vars": { + "RCCL_MSCCL_ENABLE": "0", + "SGLANG_USE_AITER": "1", + "SGLANG_INT4_WEIGHT": "1", + }, + } + + cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url) + cls.runner.setup_profile_directory() + cls.runner.full_report = f"## {cls.__name__}\n" + + def test_grok1_int4_perf(self): + """Run Grok-1 INT4 performance benchmark on MI35x.""" + # Set environment variables + old_env = {} + for key, value in self.model_config.get("env_vars", {}).items(): + old_env[key] = os.environ.get(key) + os.environ[key] = value + print(f"Setting env: {key}={value}") + + try: + result_tuple = self.runner.run_benchmark_for_model( + model_path=self.model_config["model_path"], + batch_sizes=self.batch_sizes, + input_lens=self.input_lens, + output_lens=self.output_lens, + other_args=self.model_config["other_args"], + variant=self.model_config["name"], + extra_bench_args=["--trust-remote-code"], + ) + results = result_tuple[0] + success = result_tuple[1] + + if results: + self.runner.full_report += ( + generate_simple_markdown_report(results) + "\n" + ) + + self.assertTrue(success, "Benchmark failed for Grok-1 INT4 on MI35x") + finally: + # Restore original environment + for key, value in old_env.items(): + if value is None: + os.environ.pop(key, None) + else: + os.environ[key] = value + self.runner.write_final_report() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/amd/perf/mi35x/test_grok2_perf_mi35x.py b/test/registered/amd/perf/mi35x/test_grok2_perf_mi35x.py new file mode 100644 index 000000000..ba55d1e5b --- /dev/null +++ b/test/registered/amd/perf/mi35x/test_grok2_perf_mi35x.py @@ -0,0 +1,126 @@ +"""MI35x Nightly performance benchmark for Grok-2. + +This test benchmarks Grok-2 with FP8 quantization on 8 GPUs. + +Registry: nightly-perf-8-gpu-mi35x-grok2 suite +""" + +import os +import unittest +from typing import List + +from sglang.test.ci.ci_register import register_amd_ci +from sglang.test.nightly_bench_utils import BenchmarkResult +from sglang.test.nightly_utils import NightlyBenchmarkRunner +from sglang.test.test_utils import DEFAULT_URL_FOR_TEST, _parse_int_list_env + +# Register for AMD CI - Grok-2 benchmark on MI35x (~25 min) +register_amd_ci(est_time=1500, suite="nightly-perf-8-gpu-mi35x-grok2", nightly=True) + + +def generate_simple_markdown_report(results: List[BenchmarkResult]) -> str: + """Generate a simplified markdown report without traces and cost columns.""" + model_header = results[0].model_path + if results[0].run_name and results[0].run_name != "default": + model_header += f" ({results[0].run_name})" + + gpu_config = os.getenv("GPU_CONFIG", "MI35x") + if gpu_config: + model_header += f" [{gpu_config}]" + + summary = f"### {model_header}\n" + summary += "| batch size | input len | latency (s) | input throughput (tok/s) | output throughput (tok/s) | ITL (ms) |\n" + summary += "| ---------- | --------- | ----------- | ------------------------ | ------------------------- | -------- |\n" + + for result in results: + itl = 1 / (result.output_throughput / result.batch_size) * 1000 + summary += f"| {result.batch_size} | {result.input_len} | {result.latency:.2f} | {result.input_throughput:.2f} | {result.output_throughput:.2f} | {itl:.2f} |\n" + + return summary + + +# Model and tokenizer paths can be overridden via environment variables +GROK2_MODEL_PATH = os.environ.get("GROK2_MODEL_PATH", "xai-org/grok-2") +GROK2_TOKENIZER_PATH = os.environ.get( + "GROK2_TOKENIZER_PATH", "alvarobartt/grok-2-tokenizer" +) +PROFILE_DIR = "performance_profiles_grok2_mi35x" + + +class TestGrok2PerfMI35x(unittest.TestCase): + """Test suite for Grok-2 performance benchmarks on MI35x.""" + + @classmethod + def setUpClass(cls): + cls.base_url = DEFAULT_URL_FOR_TEST + cls.batch_sizes = [1, 1, 8, 16, 64] + cls.input_lens = tuple(_parse_int_list_env("NIGHTLY_INPUT_LENS", "1024")) + cls.output_lens = tuple(_parse_int_list_env("NIGHTLY_OUTPUT_LENS", "512")) + + cls.model_config = { + "name": "grok2-mi35x", + "model_path": GROK2_MODEL_PATH, + "other_args": [ + "--trust-remote-code", + "--tp", + "8", + "--quantization", + "fp8", + "--mem-fraction-static", + "0.85", + "--tokenizer-path", + GROK2_TOKENIZER_PATH, + "--attention-backend", + "aiter", + ], + "env_vars": { + "RCCL_MSCCL_ENABLE": "0", + "SGLANG_USE_AITER": "1", + "SGLANG_INT4_WEIGHT": "0", + }, + } + + cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url) + cls.runner.setup_profile_directory() + cls.runner.full_report = f"## {cls.__name__}\n" + + def test_grok2_perf(self): + """Run Grok-2 performance benchmark on MI35x.""" + # Set environment variables + old_env = {} + for key, value in self.model_config.get("env_vars", {}).items(): + old_env[key] = os.environ.get(key) + os.environ[key] = value + print(f"Setting env: {key}={value}") + + try: + result_tuple = self.runner.run_benchmark_for_model( + model_path=self.model_config["model_path"], + batch_sizes=self.batch_sizes, + input_lens=self.input_lens, + output_lens=self.output_lens, + other_args=self.model_config["other_args"], + variant=self.model_config["name"], + extra_bench_args=["--trust-remote-code"], + ) + results = result_tuple[0] + success = result_tuple[1] + + if results: + self.runner.full_report += ( + generate_simple_markdown_report(results) + "\n" + ) + + self.assertTrue(success, "Benchmark failed for Grok-2 on MI35x") + finally: + # Restore original environment + for key, value in old_env.items(): + if value is None: + os.environ.pop(key, None) + else: + os.environ[key] = value + self.runner.write_final_report() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/amd/test_deepseek_v31_perf.py b/test/registered/amd/perf/test_deepseek_v31_perf.py similarity index 100% rename from test/registered/amd/test_deepseek_v31_perf.py rename to test/registered/amd/perf/test_deepseek_v31_perf.py diff --git a/test/registered/amd/test_deepseek_v3_perf.py b/test/registered/amd/perf/test_deepseek_v3_perf.py similarity index 100% rename from test/registered/amd/test_deepseek_v3_perf.py rename to test/registered/amd/perf/test_deepseek_v3_perf.py diff --git a/test/registered/amd/perf/test_grok1_fp8_perf.py b/test/registered/amd/perf/test_grok1_fp8_perf.py new file mode 100644 index 000000000..b74cb8e22 --- /dev/null +++ b/test/registered/amd/perf/test_grok1_fp8_perf.py @@ -0,0 +1,133 @@ +"""Nightly performance benchmark for Grok-1 FP8. + +This test benchmarks Grok-1 (314B MOE) with FP8 quantization on 8 GPUs. + +Model paths can be configured via environment variables: +- GROK1_MODEL_PATH: Path to Grok-1 model (default: lmzheng/grok-1) +- GROK1_TOKENIZER_PATH: Path to Grok-1 tokenizer (default: Xenova/grok-1-tokenizer) + +Example usage: + python -m pytest test_grok1_fp8_perf.py -v +""" + +import os +import unittest +from typing import List + +from sglang.test.ci.ci_register import register_amd_ci +from sglang.test.nightly_bench_utils import BenchmarkResult +from sglang.test.nightly_utils import NightlyBenchmarkRunner +from sglang.test.test_utils import DEFAULT_URL_FOR_TEST, _parse_int_list_env + +# Register for AMD CI - Grok-1 FP8 benchmark (~25 min) +register_amd_ci(est_time=1500, suite="nightly-perf-8-gpu-grok1-fp8", nightly=True) + + +def generate_simple_markdown_report(results: List[BenchmarkResult]) -> str: + """Generate a simplified markdown report without traces and cost columns.""" + model_header = results[0].model_path + if results[0].run_name and results[0].run_name != "default": + model_header += f" ({results[0].run_name})" + + gpu_config = os.getenv("GPU_CONFIG", "") + if gpu_config: + model_header += f" [{gpu_config}]" + + summary = f"### {model_header}\n" + summary += "| batch size | input len | latency (s) | input throughput (tok/s) | output throughput (tok/s) | ITL (ms) |\n" + summary += "| ---------- | --------- | ----------- | ------------------------ | ------------------------- | -------- |\n" + + for result in results: + itl = 1 / (result.output_throughput / result.batch_size) * 1000 + summary += f"| {result.batch_size} | {result.input_len} | {result.latency:.2f} | {result.input_throughput:.2f} | {result.output_throughput:.2f} | {itl:.2f} |\n" + + return summary + + +# Model and tokenizer paths can be overridden via environment variables +GROK1_MODEL_PATH = os.environ.get("GROK1_MODEL_PATH", "lmzheng/grok-1") +GROK1_TOKENIZER_PATH = os.environ.get("GROK1_TOKENIZER_PATH", "Xenova/grok-1-tokenizer") +PROFILE_DIR = "performance_profiles_grok1_fp8" + + +class TestNightlyGrok1FP8Performance(unittest.TestCase): + """Nightly performance benchmark for Grok-1 FP8. + + Tests Grok-1 (314B MOE) with FP8 quantization on TP=8. + Runtime: ~25 minutes + """ + + @classmethod + def setUpClass(cls): + cls.base_url = DEFAULT_URL_FOR_TEST + cls.batch_sizes = [1, 1, 8, 16, 64] + cls.input_lens = tuple(_parse_int_list_env("NIGHTLY_INPUT_LENS", "1024")) + cls.output_lens = tuple(_parse_int_list_env("NIGHTLY_OUTPUT_LENS", "512")) + + cls.model_config = { + "name": "grok1-fp8", + "model_path": GROK1_MODEL_PATH, + "other_args": [ + "--trust-remote-code", + "--tp", + "8", + "--quantization", + "fp8", + "--mem-fraction-static", + "0.85", + "--tokenizer-path", + GROK1_TOKENIZER_PATH, + "--attention-backend", + "aiter", + ], + "env_vars": { + "RCCL_MSCCL_ENABLE": "0", + "SGLANG_USE_AITER": "1", + "SGLANG_INT4_WEIGHT": "0", + }, + } + + cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url) + cls.runner.setup_profile_directory() + cls.runner.full_report = f"## {cls.__name__}\n" + + def test_bench_grok1_fp8(self): + """Run benchmark for Grok-1 FP8.""" + # Set environment variables + old_env = {} + for key, value in self.model_config.get("env_vars", {}).items(): + old_env[key] = os.environ.get(key) + os.environ[key] = value + print(f"Setting env: {key}={value}") + + try: + result_tuple = self.runner.run_benchmark_for_model( + model_path=self.model_config["model_path"], + batch_sizes=self.batch_sizes, + input_lens=self.input_lens, + output_lens=self.output_lens, + other_args=self.model_config["other_args"], + variant=self.model_config["name"], + extra_bench_args=["--trust-remote-code"], + ) + results = result_tuple[0] + success = result_tuple[1] + + if results: + self.runner.full_report += ( + generate_simple_markdown_report(results) + "\n" + ) + + self.assertTrue(success, "Benchmark failed for Grok-1 FP8") + finally: + # Restore original environment + for key, value in old_env.items(): + if value is None: + os.environ.pop(key, None) + else: + os.environ[key] = value + self.runner.write_final_report() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/amd/perf/test_grok1_int4_perf.py b/test/registered/amd/perf/test_grok1_int4_perf.py new file mode 100644 index 000000000..ed336e056 --- /dev/null +++ b/test/registered/amd/perf/test_grok1_int4_perf.py @@ -0,0 +1,133 @@ +"""Nightly performance benchmark for Grok-1 INT4 (W4A8KV8). + +This test benchmarks Grok-1 (314B MOE) with INT4 weight quantization on 8 GPUs. + +Model paths can be configured via environment variables: +- GROK1_MODEL_PATH: Path to Grok-1 INT4 model (default: amd/grok-1-W4A8KV8) +- GROK1_TOKENIZER_PATH: Path to Grok-1 tokenizer (default: Xenova/grok-1-tokenizer) + +Example usage: + python -m pytest test_grok1_int4_perf.py -v +""" + +import os +import unittest +from typing import List + +from sglang.test.ci.ci_register import register_amd_ci +from sglang.test.nightly_bench_utils import BenchmarkResult +from sglang.test.nightly_utils import NightlyBenchmarkRunner +from sglang.test.test_utils import DEFAULT_URL_FOR_TEST, _parse_int_list_env + +# Register for AMD CI - Grok-1 INT4 benchmark (~25 min) +register_amd_ci(est_time=1500, suite="nightly-perf-8-gpu-grok1-int4", nightly=True) + + +def generate_simple_markdown_report(results: List[BenchmarkResult]) -> str: + """Generate a simplified markdown report without traces and cost columns.""" + model_header = results[0].model_path + if results[0].run_name and results[0].run_name != "default": + model_header += f" ({results[0].run_name})" + + gpu_config = os.getenv("GPU_CONFIG", "") + if gpu_config: + model_header += f" [{gpu_config}]" + + summary = f"### {model_header}\n" + summary += "| batch size | input len | latency (s) | input throughput (tok/s) | output throughput (tok/s) | ITL (ms) |\n" + summary += "| ---------- | --------- | ----------- | ------------------------ | ------------------------- | -------- |\n" + + for result in results: + itl = 1 / (result.output_throughput / result.batch_size) * 1000 + summary += f"| {result.batch_size} | {result.input_len} | {result.latency:.2f} | {result.input_throughput:.2f} | {result.output_throughput:.2f} | {itl:.2f} |\n" + + return summary + + +# Model and tokenizer paths can be overridden via environment variables +GROK1_MODEL_PATH = os.environ.get("GROK1_MODEL_PATH", "amd/grok-1-W4A8KV8") +GROK1_TOKENIZER_PATH = os.environ.get("GROK1_TOKENIZER_PATH", "Xenova/grok-1-tokenizer") +PROFILE_DIR = "performance_profiles_grok1_int4" + + +class TestNightlyGrok1INT4Performance(unittest.TestCase): + """Nightly performance benchmark for Grok-1 INT4 (W4A8KV8). + + Tests Grok-1 (314B MOE) with INT4 weight quantization on TP=8. + Runtime: ~25 minutes + """ + + @classmethod + def setUpClass(cls): + cls.base_url = DEFAULT_URL_FOR_TEST + cls.batch_sizes = [1, 1, 8, 16, 64] + cls.input_lens = tuple(_parse_int_list_env("NIGHTLY_INPUT_LENS", "1024")) + cls.output_lens = tuple(_parse_int_list_env("NIGHTLY_OUTPUT_LENS", "512")) + + cls.model_config = { + "name": "grok1-int4", + "model_path": GROK1_MODEL_PATH, + "other_args": [ + "--trust-remote-code", + "--tp", + "8", + "--quantization", + "fp8", + "--mem-fraction-static", + "0.85", + "--tokenizer-path", + GROK1_TOKENIZER_PATH, + "--attention-backend", + "aiter", + ], + "env_vars": { + "RCCL_MSCCL_ENABLE": "0", + "SGLANG_USE_AITER": "1", + "SGLANG_INT4_WEIGHT": "1", + }, + } + + cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url) + cls.runner.setup_profile_directory() + cls.runner.full_report = f"## {cls.__name__}\n" + + def test_bench_grok1_int4(self): + """Run benchmark for Grok-1 INT4.""" + # Set environment variables + old_env = {} + for key, value in self.model_config.get("env_vars", {}).items(): + old_env[key] = os.environ.get(key) + os.environ[key] = value + print(f"Setting env: {key}={value}") + + try: + result_tuple = self.runner.run_benchmark_for_model( + model_path=self.model_config["model_path"], + batch_sizes=self.batch_sizes, + input_lens=self.input_lens, + output_lens=self.output_lens, + other_args=self.model_config["other_args"], + variant=self.model_config["name"], + extra_bench_args=["--trust-remote-code"], + ) + results = result_tuple[0] + success = result_tuple[1] + + if results: + self.runner.full_report += ( + generate_simple_markdown_report(results) + "\n" + ) + + self.assertTrue(success, "Benchmark failed for Grok-1 INT4") + finally: + # Restore original environment + for key, value in old_env.items(): + if value is None: + os.environ.pop(key, None) + else: + os.environ[key] = value + self.runner.write_final_report() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/amd/perf/test_grok2_perf.py b/test/registered/amd/perf/test_grok2_perf.py new file mode 100644 index 000000000..ed163e2f3 --- /dev/null +++ b/test/registered/amd/perf/test_grok2_perf.py @@ -0,0 +1,135 @@ +"""Nightly performance benchmark for Grok-2. + +This test benchmarks Grok-2 with FP8 quantization on 8 GPUs. + +Model paths can be configured via environment variables: +- GROK2_MODEL_PATH: Path to Grok-2 model (default: xai-org/grok-2) +- GROK2_TOKENIZER_PATH: Path to Grok-2 tokenizer (default: alvarobartt/grok-2-tokenizer) + +Example usage: + python -m pytest test_grok2_perf.py -v +""" + +import os +import unittest +from typing import List + +from sglang.test.ci.ci_register import register_amd_ci +from sglang.test.nightly_bench_utils import BenchmarkResult +from sglang.test.nightly_utils import NightlyBenchmarkRunner +from sglang.test.test_utils import DEFAULT_URL_FOR_TEST, _parse_int_list_env + +# Register for AMD CI - Grok-2 benchmark (~25 min) +register_amd_ci(est_time=1500, suite="nightly-perf-8-gpu-grok2", nightly=True) + + +def generate_simple_markdown_report(results: List[BenchmarkResult]) -> str: + """Generate a simplified markdown report without traces and cost columns.""" + model_header = results[0].model_path + if results[0].run_name and results[0].run_name != "default": + model_header += f" ({results[0].run_name})" + + gpu_config = os.getenv("GPU_CONFIG", "") + if gpu_config: + model_header += f" [{gpu_config}]" + + summary = f"### {model_header}\n" + summary += "| batch size | input len | latency (s) | input throughput (tok/s) | output throughput (tok/s) | ITL (ms) |\n" + summary += "| ---------- | --------- | ----------- | ------------------------ | ------------------------- | -------- |\n" + + for result in results: + itl = 1 / (result.output_throughput / result.batch_size) * 1000 + summary += f"| {result.batch_size} | {result.input_len} | {result.latency:.2f} | {result.input_throughput:.2f} | {result.output_throughput:.2f} | {itl:.2f} |\n" + + return summary + + +# Model and tokenizer paths can be overridden via environment variables +GROK2_MODEL_PATH = os.environ.get("GROK2_MODEL_PATH", "xai-org/grok-2") +GROK2_TOKENIZER_PATH = os.environ.get( + "GROK2_TOKENIZER_PATH", "alvarobartt/grok-2-tokenizer" +) +PROFILE_DIR = "performance_profiles_grok2" + + +class TestNightlyGrok2Performance(unittest.TestCase): + """Nightly performance benchmark for Grok-2. + + Tests Grok-2 with FP8 quantization on TP=8. + Runtime: ~25 minutes + """ + + @classmethod + def setUpClass(cls): + cls.base_url = DEFAULT_URL_FOR_TEST + cls.batch_sizes = [1, 1, 8, 16, 64] + cls.input_lens = tuple(_parse_int_list_env("NIGHTLY_INPUT_LENS", "1024")) + cls.output_lens = tuple(_parse_int_list_env("NIGHTLY_OUTPUT_LENS", "512")) + + cls.model_config = { + "name": "grok2", + "model_path": GROK2_MODEL_PATH, + "other_args": [ + "--trust-remote-code", + "--tp", + "8", + "--quantization", + "fp8", + "--mem-fraction-static", + "0.85", + "--tokenizer-path", + GROK2_TOKENIZER_PATH, + "--attention-backend", + "aiter", + ], + "env_vars": { + "RCCL_MSCCL_ENABLE": "0", + "SGLANG_USE_AITER": "1", + "SGLANG_INT4_WEIGHT": "0", + }, + } + + cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url) + cls.runner.setup_profile_directory() + cls.runner.full_report = f"## {cls.__name__}\n" + + def test_bench_grok2(self): + """Run benchmark for Grok-2.""" + # Set environment variables + old_env = {} + for key, value in self.model_config.get("env_vars", {}).items(): + old_env[key] = os.environ.get(key) + os.environ[key] = value + print(f"Setting env: {key}={value}") + + try: + result_tuple = self.runner.run_benchmark_for_model( + model_path=self.model_config["model_path"], + batch_sizes=self.batch_sizes, + input_lens=self.input_lens, + output_lens=self.output_lens, + other_args=self.model_config["other_args"], + variant=self.model_config["name"], + extra_bench_args=["--trust-remote-code"], + ) + results = result_tuple[0] + success = result_tuple[1] + + if results: + self.runner.full_report += ( + generate_simple_markdown_report(results) + "\n" + ) + + self.assertTrue(success, "Benchmark failed for Grok-2") + finally: + # Restore original environment + for key, value in old_env.items(): + if value is None: + os.environ.pop(key, None) + else: + os.environ[key] = value + self.runner.write_final_report() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/amd/test_grok_perf.py b/test/registered/amd/test_grok_perf.py deleted file mode 100644 index 5e18e2af7..000000000 --- a/test/registered/amd/test_grok_perf.py +++ /dev/null @@ -1,178 +0,0 @@ -"""Nightly performance benchmark for Grok models (Grok-1 and Grok-2). - -This test benchmarks both Grok-1 and Grok-2 models with FP8 quantization on 8 GPUs. - -Model paths can be configured via environment variables: -- GROK1_MODEL_PATH: Path to Grok-1 model (default: amd/grok-1-W4A8KV8) -- GROK1_TOKENIZER_PATH: Path to Grok-1 tokenizer (default: Xenova/grok-1-tokenizer) -- GROK2_MODEL_PATH: Path to Grok-2 model (default: xai-org/grok-2) -- GROK2_TOKENIZER_PATH: Path to Grok-2 tokenizer (default: alvarobartt/grok-2-tokenizer) - -Example usage: - python -m pytest test_grok_perf.py -v -""" - -import os -import unittest -from typing import List - -from sglang.test.ci.ci_register import register_amd_ci -from sglang.test.nightly_bench_utils import BenchmarkResult -from sglang.test.nightly_utils import NightlyBenchmarkRunner -from sglang.test.test_utils import DEFAULT_URL_FOR_TEST, _parse_int_list_env - -# Register for AMD CI - combined Grok-1 + Grok-2 benchmark (~60 min) -register_amd_ci(est_time=3600, suite="nightly-perf-8-gpu-grok", nightly=True) - - -def generate_simple_markdown_report(results: List[BenchmarkResult]) -> str: - """Generate a simplified markdown report without traces and cost columns.""" - model_header = results[0].model_path - if results[0].run_name and results[0].run_name != "default": - model_header += f" ({results[0].run_name})" - - gpu_config = os.getenv("GPU_CONFIG", "") - if gpu_config: - model_header += f" [{gpu_config}]" - - summary = f"### {model_header}\n" - summary += "| batch size | input len | latency (s) | input throughput (tok/s) | output throughput (tok/s) | ITL (ms) |\n" - summary += "| ---------- | --------- | ----------- | ------------------------ | ------------------------- | -------- |\n" - - for result in results: - itl = 1 / (result.output_throughput / result.batch_size) * 1000 - summary += f"| {result.batch_size} | {result.input_len} | {result.latency:.2f} | {result.input_throughput:.2f} | {result.output_throughput:.2f} | {itl:.2f} |\n" - - return summary - - -# Model and tokenizer paths can be overridden via environment variables -GROK1_MODEL_PATH = os.environ.get("GROK1_MODEL_PATH", "amd/grok-1-W4A8KV8") -GROK1_TOKENIZER_PATH = os.environ.get("GROK1_TOKENIZER_PATH", "Xenova/grok-1-tokenizer") -GROK2_MODEL_PATH = os.environ.get("GROK2_MODEL_PATH", "xai-org/grok-2") -GROK2_TOKENIZER_PATH = os.environ.get( - "GROK2_TOKENIZER_PATH", "alvarobartt/grok-2-tokenizer" -) -PROFILE_DIR = "performance_profiles_grok" - - -class TestNightlyGrokPerformance(unittest.TestCase): - """Nightly performance benchmark for Grok models (Grok-1 and Grok-2). - - Tests both Grok-1 (314B MOE) and Grok-2 models with FP8 quantization on TP=8. - Combined runtime: ~43 minutes (Grok-1: ~23min, Grok-2: ~20min) - """ - - @classmethod - def setUpClass(cls): - cls.base_url = DEFAULT_URL_FOR_TEST - cls.batch_sizes = [1, 1, 8, 16, 64] - cls.input_lens = tuple(_parse_int_list_env("NIGHTLY_INPUT_LENS", "1024")) - cls.output_lens = tuple(_parse_int_list_env("NIGHTLY_OUTPUT_LENS", "512")) - - # Define model configurations for both Grok-1 and Grok-2 - cls.models = [ - { - "name": "grok1", - "model_path": GROK1_MODEL_PATH, - "other_args": [ - "--trust-remote-code", - "--tp", - "8", - "--quantization", - "fp8", - "--mem-fraction-static", - "0.85", - "--tokenizer-path", - GROK1_TOKENIZER_PATH, - "--attention-backend", - "aiter", - ], - "env_vars": { - "RCCL_MSCCL_ENABLE": "0", - "SGLANG_USE_AITER": "1", - "SGLANG_INT4_WEIGHT": "1", - }, - }, - { - "name": "grok2", - "model_path": GROK2_MODEL_PATH, - "other_args": [ - "--trust-remote-code", - "--tp", - "8", - "--quantization", - "fp8", - "--mem-fraction-static", - "0.85", - "--tokenizer-path", - GROK2_TOKENIZER_PATH, - "--attention-backend", - "aiter", - ], - "env_vars": { - "RCCL_MSCCL_ENABLE": "0", - "SGLANG_USE_AITER": "1", - "SGLANG_INT4_WEIGHT": "0", - }, - }, - ] - - cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url) - cls.runner.setup_profile_directory() - # Override full_report to remove traces help text - cls.runner.full_report = f"## {cls.__name__}\n" - - def test_bench_one_batch(self): - """Run benchmark across all Grok models.""" - failed_models = [] - - try: - for model_config in self.models: - with self.subTest(model=model_config["name"]): - # Set environment variables for this model - old_env = {} - for key, value in model_config.get("env_vars", {}).items(): - old_env[key] = os.environ.get(key) - os.environ[key] = value - print(f"Setting env: {key}={value}") - - try: - result_tuple = self.runner.run_benchmark_for_model( - model_path=model_config["model_path"], - batch_sizes=self.batch_sizes, - input_lens=self.input_lens, - output_lens=self.output_lens, - other_args=model_config["other_args"], - variant=model_config["name"], - extra_bench_args=["--trust-remote-code"], - ) - results = result_tuple[0] - success = result_tuple[1] - - if not success: - failed_models.append(model_config["name"]) - - # Use simplified report format without traces - if results: - self.runner.full_report += ( - generate_simple_markdown_report(results) + "\n" - ) - finally: - # Restore original environment - for key, value in old_env.items(): - if value is None: - os.environ.pop(key, None) - else: - os.environ[key] = value - finally: - self.runner.write_final_report() - - if failed_models: - raise AssertionError( - f"Benchmark failed for the following models: {', '.join(failed_models)}" - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/test/registered/bench_fn/test_bench_serving_functionality.py b/test/registered/bench_fn/test_bench_serving_functionality.py index 69557fc98..ab403f626 100644 --- a/test/registered/bench_fn/test_bench_serving_functionality.py +++ b/test/registered/bench_fn/test_bench_serving_functionality.py @@ -8,7 +8,7 @@ from pathlib import Path from sglang.bench_serving import parse_custom_headers, run_benchmark from sglang.srt.utils import kill_process_tree -from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, DEFAULT_URL_FOR_TEST, @@ -18,6 +18,7 @@ from sglang.test.test_utils import ( ) register_cuda_ci(est_time=300, suite="nightly-1-gpu", nightly=True) +register_amd_ci(est_time=300, suite="nightly-amd-1-gpu", nightly=True) MODEL = "Qwen/Qwen3-0.6B" NUM_CONVERSATIONS, NUM_TURNS = 4, 3 diff --git a/test/registered/core/test_batch_invariant_ops.py b/test/registered/core/test_batch_invariant_ops.py index d72feabc0..e17d0e588 100644 --- a/test/registered/core/test_batch_invariant_ops.py +++ b/test/registered/core/test_batch_invariant_ops.py @@ -5,12 +5,15 @@ import unittest import torch from sglang.srt.batch_invariant_ops import batch_invariant_ops -from sglang.test.ci.ci_register import register_cuda_ci - -register_cuda_ci(est_time=10, suite="nightly-1-gpu", nightly=True) from sglang.srt.batch_invariant_ops.batch_invariant_ops import set_batch_invariant_mode +from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci from sglang.test.test_utils import CustomTestCase +# Note: MI300 (gfx942) has 64KB shared memory limit but kernel needs 66KB +# MI35x (gfx950/CDNA4) may have different limits - testing on MI35x only +register_cuda_ci(est_time=10, suite="nightly-1-gpu", nightly=True) +register_amd_ci(est_time=10, suite="nightly-amd-1-gpu-mi35x", nightly=True) + device_type = getattr(torch.accelerator.current_accelerator(), "type", "cpu") torch.set_default_device(device_type) diff --git a/test/registered/core/test_cpp_radix_cache.py b/test/registered/core/test_cpp_radix_cache.py index 40ab477e9..faf7d2d45 100644 --- a/test/registered/core/test_cpp_radix_cache.py +++ b/test/registered/core/test_cpp_radix_cache.py @@ -13,6 +13,7 @@ from sglang.test.test_utils import ( popen_launch_server, ) +# Note: AMD registration removed - test_cpp_radix_cache fails on AMD due to C++ radix tree issues register_cuda_ci(est_time=60, suite="nightly-1-gpu", nightly=True) diff --git a/test/registered/debug_utils/test_crash_dump.py b/test/registered/debug_utils/test_crash_dump.py index a3cb4df80..0461a4861 100644 --- a/test/registered/debug_utils/test_crash_dump.py +++ b/test/registered/debug_utils/test_crash_dump.py @@ -9,7 +9,7 @@ import requests from sglang.srt.environ import envs from sglang.srt.utils import kill_process_tree -from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, DEFAULT_URL_FOR_TEST, @@ -18,6 +18,7 @@ from sglang.test.test_utils import ( ) register_cuda_ci(est_time=40, suite="nightly-1-gpu", nightly=True) +register_amd_ci(est_time=40, suite="nightly-amd-1-gpu", nightly=True) class TestCrashDump(CustomTestCase): diff --git a/test/registered/debug_utils/test_dumper.py b/test/registered/debug_utils/test_dumper.py index 6eb52b800..b69a5e626 100644 --- a/test/registered/debug_utils/test_dumper.py +++ b/test/registered/debug_utils/test_dumper.py @@ -9,10 +9,11 @@ import torch import torch.distributed as dist import torch.multiprocessing as mp -from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci from sglang.test.test_utils import CustomTestCase register_cuda_ci(est_time=30, suite="nightly-2-gpu", nightly=True) +register_amd_ci(est_time=60, suite="nightly-amd", nightly=True) class TestDumperPureFunctions(CustomTestCase): diff --git a/test/registered/debug_utils/test_soft_watchdog.py b/test/registered/debug_utils/test_soft_watchdog.py index dfd523adf..c18efda19 100644 --- a/test/registered/debug_utils/test_soft_watchdog.py +++ b/test/registered/debug_utils/test_soft_watchdog.py @@ -5,7 +5,7 @@ import requests from sglang.srt.environ import envs from sglang.srt.utils import kill_process_tree -from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, DEFAULT_URL_FOR_TEST, @@ -14,6 +14,7 @@ from sglang.test.test_utils import ( ) register_cuda_ci(est_time=120, suite="nightly-1-gpu", nightly=True) +register_amd_ci(est_time=120, suite="nightly-amd-1-gpu", nightly=True) class BaseTestSoftWatchdog: diff --git a/test/registered/lora/test_lora_eviction_policy.py b/test/registered/lora/test_lora_eviction_policy.py index 073df2542..4d0f5cd3e 100644 --- a/test/registered/lora/test_lora_eviction_policy.py +++ b/test/registered/lora/test_lora_eviction_policy.py @@ -20,9 +20,10 @@ Tests LRU and FIFO eviction behavior. import unittest from sglang.srt.lora.eviction_policy import get_eviction_policy -from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci register_cuda_ci(est_time=200, suite="nightly-1-gpu", nightly=True) +register_amd_ci(est_time=200, suite="nightly-amd-1-gpu", nightly=True) class TestLoRAEvictionPolicy(unittest.TestCase): diff --git a/test/registered/lora/test_lora_openai_api.py b/test/registered/lora/test_lora_openai_api.py index b0f856493..140d95816 100644 --- a/test/registered/lora/test_lora_openai_api.py +++ b/test/registered/lora/test_lora_openai_api.py @@ -9,10 +9,11 @@ import unittest from unittest.mock import MagicMock from sglang.srt.entrypoints.openai.serving_base import OpenAIServingBase -from sglang.test.ci.ci_register import register_cuda_ci +from sglang.srt.server_args import ServerArgs +from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci register_cuda_ci(est_time=30, suite="nightly-1-gpu", nightly=True) -from sglang.srt.server_args import ServerArgs +register_amd_ci(est_time=30, suite="nightly-amd-1-gpu", nightly=True) class MockTokenizerManager: diff --git a/test/registered/lora/test_lora_openai_compatible.py b/test/registered/lora/test_lora_openai_compatible.py index 5cb212993..b56133386 100644 --- a/test/registered/lora/test_lora_openai_compatible.py +++ b/test/registered/lora/test_lora_openai_compatible.py @@ -17,9 +17,10 @@ import unittest import openai -from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci register_cuda_ci(est_time=150, suite="nightly-1-gpu", nightly=True) +register_amd_ci(est_time=150, suite="nightly-amd-1-gpu", nightly=True) from sglang.srt.utils import kill_process_tree from sglang.test.test_utils import ( diff --git a/test/registered/lora/test_lora_radix_cache.py b/test/registered/lora/test_lora_radix_cache.py index 84ffd2cce..d59572ed8 100644 --- a/test/registered/lora/test_lora_radix_cache.py +++ b/test/registered/lora/test_lora_radix_cache.py @@ -17,12 +17,12 @@ import unittest import torch -from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci from sglang.test.lora_utils import CI_MULTI_LORA_MODELS, run_lora_test_one_by_one +from sglang.test.test_utils import CustomTestCase register_cuda_ci(est_time=200, suite="nightly-1-gpu", nightly=True) - -from sglang.test.test_utils import CustomTestCase +register_amd_ci(est_time=200, suite="nightly-amd-1-gpu", nightly=True) PROMPTS = [ "AI is a field of computer science focused on", diff --git a/test/registered/scheduler/test_routing_key_scheduling.py b/test/registered/scheduler/test_routing_key_scheduling.py index be4a7d30e..f08a56c18 100644 --- a/test/registered/scheduler/test_routing_key_scheduling.py +++ b/test/registered/scheduler/test_routing_key_scheduling.py @@ -6,7 +6,7 @@ import unittest import aiohttp from sglang.srt.utils import kill_process_tree -from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, DEFAULT_URL_FOR_TEST, @@ -17,6 +17,7 @@ from sglang.test.test_utils import ( ) register_cuda_ci(est_time=120, suite="nightly-1-gpu", nightly=True) +register_amd_ci(est_time=120, suite="nightly-amd-1-gpu", nightly=True) class TestRoutingKeyScheduling(CustomTestCase): diff --git a/test/registered/utils/test_model_file_verifier.py b/test/registered/utils/test_model_file_verifier.py index 4c2f3d4c8..11455ba4e 100644 --- a/test/registered/utils/test_model_file_verifier.py +++ b/test/registered/utils/test_model_file_verifier.py @@ -27,6 +27,7 @@ from sglang.test.test_utils import ( popen_launch_server, ) +# Note: AMD registration removed - test_model_file_verifier fails on AMD register_cuda_ci(est_time=120, suite="nightly-1-gpu", nightly=True) MODEL_NAME = "Qwen/Qwen3-0.6B" diff --git a/test/registered/utils/test_request_logger.py b/test/registered/utils/test_request_logger.py index a9b04637d..a7744b758 100644 --- a/test/registered/utils/test_request_logger.py +++ b/test/registered/utils/test_request_logger.py @@ -8,7 +8,7 @@ from pathlib import Path import requests from sglang.srt.utils import kill_process_tree -from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, DEFAULT_URL_FOR_TEST, @@ -17,6 +17,7 @@ from sglang.test.test_utils import ( ) register_cuda_ci(est_time=120, suite="nightly-1-gpu", nightly=True) +register_amd_ci(est_time=120, suite="nightly-amd-1-gpu", nightly=True) TEST_ROUTING_KEY = "test-routing-key-12345" diff --git a/test/registered/utils/test_scheduler_status_logger.py b/test/registered/utils/test_scheduler_status_logger.py index 3102e159e..1e366d008 100644 --- a/test/registered/utils/test_scheduler_status_logger.py +++ b/test/registered/utils/test_scheduler_status_logger.py @@ -9,7 +9,7 @@ from pathlib import Path import requests from sglang.srt.utils import kill_process_tree -from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, DEFAULT_URL_FOR_TEST, @@ -18,6 +18,7 @@ from sglang.test.test_utils import ( ) register_cuda_ci(est_time=120, suite="nightly-1-gpu", nightly=True) +register_amd_ci(est_time=120, suite="nightly-amd-1-gpu", nightly=True) class TestSchedulerStatusLogger(CustomTestCase):