chenchenghao/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 11 Packages Projects Releases Wiki Activity

feature: revamp nightly tests with combined runner (#15324)

Browse Source
This commit is contained in:
Douglas Yang
2025-12-20 19:25:22 -08:00
committed by GitHub
parent 9a3bdf2c95
commit 96740d6983
23 changed files with 1332 additions and 1420 deletions
Download Patch File Download Diff File Expand all files Collapse all files

225
.github/workflows/nightly-test-nvidia.yml vendored
View File

@@ -108,6 +108,17 @@ jobs:
run: |
bash scripts/ci/ci_install_dependency.sh
- name: Run common 8-GPU model tests
if: always()
timeout-minutes: 200
env:
TRACE_BASE_URL: https://raw.githubusercontent.com/sglang-bot/sglang-ci-data/main/traces/${{ github.run_id }}
PERFETTO_RELAY_URL: ${{ vars.PERFETTO_RELAY_URL }}
GPU_CONFIG: "8-gpu-h200"
run: |
cd test
python3 run_suite.py --hw cuda --suite nightly-8-gpu-common --nightly --timeout-per-file=12000 --continue-on-error
- name: Run test
timeout-minutes: 30
env:
@@ -116,82 +127,6 @@ jobs:
cd test
python3 run_suite.py --hw cuda --suite nightly-8-gpu-h200 --nightly --continue-on-error
- name: Run MiniMax-M2 nightly performance test
timeout-minutes: 180
env:
TRACE_BASE_URL: https://raw.githubusercontent.com/sglang-bot/sglang-ci-data/main/traces/${{ github.run_id }}
PERFETTO_RELAY_URL: ${{ vars.PERFETTO_RELAY_URL }}
GPU_CONFIG: "8-gpu-h200"
run: |
rm -rf test/performance_profiles_minimax_m2/
cd test
python3 nightly/test_minimax_m2_perf.py
- name: Publish MiniMax-M2 traces to storage repo
env:
GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }}
GITHUB_RUN_ID: ${{ github.run_id }}
GITHUB_RUN_NUMBER: ${{ github.run_number }}
run: |
python3 scripts/ci/publish_traces.py --traces-dir test/performance_profiles_minimax_m2
- name: Run Qwen3-235B nightly performance test
timeout-minutes: 180
env:
TRACE_BASE_URL: https://raw.githubusercontent.com/sglang-bot/sglang-ci-data/main/traces/${{ github.run_id }}
PERFETTO_RELAY_URL: ${{ vars.PERFETTO_RELAY_URL }}
GPU_CONFIG: "8-gpu-h200"
run: |
rm -rf test/performance_profiles_qwen3_235b/
cd test
python3 nightly/test_qwen3_235b_perf.py
- name: Publish Qwen3-235B traces to storage repo
env:
GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }}
GITHUB_RUN_ID: ${{ github.run_id }}
GITHUB_RUN_NUMBER: ${{ github.run_number }}
run: |
python3 scripts/ci/publish_traces.py --traces-dir test/performance_profiles_qwen3_235b
- name: Run Kimi-K2-Thinking nightly performance test
timeout-minutes: 180
env:
TRACE_BASE_URL: https://raw.githubusercontent.com/sglang-bot/sglang-ci-data/main/traces/${{ github.run_id }}
PERFETTO_RELAY_URL: ${{ vars.PERFETTO_RELAY_URL }}
GPU_CONFIG: "8-gpu-h200"
run: |
rm -rf test/performance_profiles_kimi_k2_thinking/
cd test
python3 nightly/test_kimi_k2_thinking_perf.py
- name: Publish Kimi-K2-Thinking traces to storage repo
env:
GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }}
GITHUB_RUN_ID: ${{ github.run_id }}
GITHUB_RUN_NUMBER: ${{ github.run_number }}
run: |
python3 scripts/ci/publish_traces.py --traces-dir test/performance_profiles_kimi_k2_thinking
- name: Run GLM-4.6 nightly performance test
timeout-minutes: 180
env:
TRACE_BASE_URL: https://raw.githubusercontent.com/sglang-bot/sglang-ci-data/main/traces/${{ github.run_id }}
PERFETTO_RELAY_URL: ${{ vars.PERFETTO_RELAY_URL }}
GPU_CONFIG: "8-gpu-h200"
run: |
rm -rf test/performance_profiles_glm_4_6/
cd test
IS_BLACKWELL=1 python3 nightly/test_glm_4_6_perf.py
- name: Publish GLM-4.6 traces to storage repo
env:
GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }}
GITHUB_RUN_ID: ${{ github.run_id }}
GITHUB_RUN_NUMBER: ${{ github.run_number }}
run: |
python3 scripts/ci/publish_traces.py --traces-dir test/performance_profiles_glm_4_6
# General tests - 8 GPU H20
nightly-test-general-8-gpu-h20:
if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-general-8-gpu-h20')
@@ -446,147 +381,17 @@ jobs:
run: |
IS_BLACKWELL=1 bash scripts/ci/ci_install_dependency.sh
- name: Run Mistral-Large-3 nightly performance test
timeout-minutes: 180
- name: Run common 8-GPU model tests
if: always()
timeout-minutes: 200
env:
TRACE_BASE_URL: https://raw.githubusercontent.com/sglang-bot/sglang-ci-data/main/traces/${{ github.run_id }}
PERFETTO_RELAY_URL: ${{ vars.PERFETTO_RELAY_URL }}
GPU_CONFIG: "8-gpu-b200"
SGLANG_ENABLE_JIT_DEEPGEMM: "0"
run: |
rm -rf test/performance_profiles_mistral_large3/
rm -rf test/performance_profiles_mistral_large3_eagle/
cd test
IS_BLACKWELL=1 python3 nightly/test_mistral_large3_perf.py
- name: Publish Mistral-Large-3 traces to storage repo
env:
GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }}
GITHUB_RUN_ID: ${{ github.run_id }}
GITHUB_RUN_NUMBER: ${{ github.run_number }}
run: |
python3 scripts/ci/publish_traces.py --traces-dir test/performance_profiles_mistral_large3
python3 scripts/ci/publish_traces.py --traces-dir test/performance_profiles_mistral_large3_eagle
- name: Run DeepSeek v3.1 nightly performance test
if: always()
timeout-minutes: 180
env:
TRACE_BASE_URL: https://raw.githubusercontent.com/sglang-bot/sglang-ci-data/main/traces/${{ github.run_id }}
PERFETTO_RELAY_URL: ${{ vars.PERFETTO_RELAY_URL }}
GPU_CONFIG: "8-gpu-b200"
run: |
rm -rf test/performance_profiles_deepseek_v31/
cd test
IS_BLACKWELL=1 python3 nightly/test_deepseek_v31_perf.py
- name: Publish DeepSeek v3.1 traces to storage repo
env:
GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }}
GITHUB_RUN_ID: ${{ github.run_id }}
GITHUB_RUN_NUMBER: ${{ github.run_number }}
run: |
python3 scripts/ci/publish_traces.py --traces-dir test/performance_profiles_deepseek_v31
- name: Run DeepSeek v3.2 nightly performance test
if: always()
timeout-minutes: 180
env:
TRACE_BASE_URL: https://raw.githubusercontent.com/sglang-bot/sglang-ci-data/main/traces/${{ github.run_id }}
PERFETTO_RELAY_URL: ${{ vars.PERFETTO_RELAY_URL }}
GPU_CONFIG: "8-gpu-b200"
run: |
rm -rf test/performance_profiles_deepseek_v32/
cd test
IS_BLACKWELL=1 python3 nightly/test_deepseek_v32_perf.py
- name: Publish DeepSeek v3.2 traces to storage repo
env:
GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }}
GITHUB_RUN_ID: ${{ github.run_id }}
GITHUB_RUN_NUMBER: ${{ github.run_number }}
run: |
python3 scripts/ci/publish_traces.py --traces-dir test/performance_profiles_deepseek_v32
- name: Run Kimi-K2-Thinking nightly performance test
if: always()
timeout-minutes: 180
env:
TRACE_BASE_URL: https://raw.githubusercontent.com/sglang-bot/sglang-ci-data/main/traces/${{ github.run_id }}
PERFETTO_RELAY_URL: ${{ vars.PERFETTO_RELAY_URL }}
GPU_CONFIG: "8-gpu-b200"
run: |
rm -rf test/performance_profiles_kimi_k2_thinking/
cd test
IS_BLACKWELL=1 python3 nightly/test_kimi_k2_thinking_perf.py
- name: Publish Kimi-K2-Thinking traces to storage repo
env:
GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }}
GITHUB_RUN_ID: ${{ github.run_id }}
GITHUB_RUN_NUMBER: ${{ github.run_number }}
run: |
python3 scripts/ci/publish_traces.py --traces-dir test/performance_profiles_kimi_k2_thinking
- name: Run Qwen3-235B nightly performance test
if: always()
timeout-minutes: 180
env:
TRACE_BASE_URL: https://raw.githubusercontent.com/sglang-bot/sglang-ci-data/main/traces/${{ github.run_id }}
PERFETTO_RELAY_URL: ${{ vars.PERFETTO_RELAY_URL }}
GPU_CONFIG: "8-gpu-b200"
run: |
rm -rf test/performance_profiles_qwen3_235b/
cd test
IS_BLACKWELL=1 python3 nightly/test_qwen3_235b_perf.py
- name: Publish Qwen3-235B traces to storage repo
env:
GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }}
GITHUB_RUN_ID: ${{ github.run_id }}
GITHUB_RUN_NUMBER: ${{ github.run_number }}
run: |
python3 scripts/ci/publish_traces.py --traces-dir test/performance_profiles_qwen3_235b
- name: Run GLM-4.6 nightly performance test
if: always()
timeout-minutes: 180
env:
TRACE_BASE_URL: https://raw.githubusercontent.com/sglang-bot/sglang-ci-data/main/traces/${{ github.run_id }}
PERFETTO_RELAY_URL: ${{ vars.PERFETTO_RELAY_URL }}
GPU_CONFIG: "8-gpu-b200"
run: |
rm -rf test/performance_profiles_glm_4_6/
cd test
IS_BLACKWELL=1 python3 nightly/test_glm_4_6_perf.py
- name: Publish GLM-4.6 traces to storage repo
env:
GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }}
GITHUB_RUN_ID: ${{ github.run_id }}
GITHUB_RUN_NUMBER: ${{ github.run_number }}
run: |
python3 scripts/ci/publish_traces.py --traces-dir test/performance_profiles_glm_4_6
- name: Run MiniMax-M2 nightly performance test
if: always()
timeout-minutes: 180
env:
TRACE_BASE_URL: https://raw.githubusercontent.com/sglang-bot/sglang-ci-data/main/traces/${{ github.run_id }}
PERFETTO_RELAY_URL: ${{ vars.PERFETTO_RELAY_URL }}
GPU_CONFIG: "8-gpu-b200"
run: |
rm -rf test/performance_profiles_minimax_m2/
cd test
IS_BLACKWELL=1 python3 nightly/test_minimax_m2_perf.py
- name: Publish MiniMax-M2 traces to storage repo
env:
GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }}
GITHUB_RUN_ID: ${{ github.run_id }}
GITHUB_RUN_NUMBER: ${{ github.run_number }}
run: |
python3 scripts/ci/publish_traces.py --traces-dir test/performance_profiles_minimax_m2
IS_BLACKWELL=1 python3 run_suite.py --hw cuda --suite nightly-8-gpu-common --nightly --timeout-per-file=12000 --continue-on-error
# Final check job
check-all-jobs: