diff --git a/.github/workflows/weekly-test-nvidia.yml b/.github/workflows/weekly-test-nvidia.yml new file mode 100644 index 000000000..c51b8d7ee --- /dev/null +++ b/.github/workflows/weekly-test-nvidia.yml @@ -0,0 +1,48 @@ +name: Weekly Test (Nvidia) + +on: + schedule: + - cron: '0 0 * * 0' # Run every Sunday at midnight UTC + workflow_dispatch: + inputs: + job_filter: + description: 'Select which job to run (leave empty or "all" to run all jobs)' + required: false + type: choice + default: 'all' + options: + - 'all' + - 'weekly-test-8-gpu-h200' + +concurrency: + group: weekly-test-nvidia-${{ github.ref }} + cancel-in-progress: true + +env: + SGLANG_IS_IN_CI: true + HF_HUB_DOWNLOAD_TIMEOUT: 300 + HF_HUB_ETAG_TIMEOUT: 300 + +jobs: + # Weekly tests - 8 GPU H200 + weekly-test-8-gpu-h200: + if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'weekly-test-8-gpu-h200') + runs-on: 8-gpu-h200 + env: + RUNNER_LABELS: 8-gpu-h200 + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Install dependencies + run: | + bash scripts/ci/ci_install_dependency.sh + + - name: Run weekly 8-GPU H200 tests + timeout-minutes: 120 + env: + GPU_CONFIG: "8-gpu-h200" + IS_H200: "1" + run: | + cd test + python3 run_suite.py --hw cuda --suite weekly-8-gpu-h200 --nightly --continue-on-error diff --git a/test/manual/ep/test_hybrid_dp_ep_tp_mtp.py b/test/registered/test_hybrid_dp_ep_tp_mtp.py similarity index 99% rename from test/manual/ep/test_hybrid_dp_ep_tp_mtp.py rename to test/registered/test_hybrid_dp_ep_tp_mtp.py index d81a977f1..06e73443d 100644 --- a/test/manual/ep/test_hybrid_dp_ep_tp_mtp.py +++ b/test/registered/test_hybrid_dp_ep_tp_mtp.py @@ -1,10 +1,10 @@ # Comprehensive test for hybrid parallelism (DP/TP attention, DP/TP Dense FFN, TP/EP Sparse FFN, DP/VP LM head) plus speculative decoding. -# These tests are not run by default but can be launched on demand. import unittest from types import SimpleNamespace from sglang.srt.utils import kill_process_tree +from sglang.test.ci.ci_register import register_cuda_ci from sglang.test.run_eval import run_eval from sglang.test.test_utils import ( DEFAULT_DEEPEP_MODEL_NAME_FOR_TEST, @@ -15,6 +15,10 @@ from sglang.test.test_utils import ( popen_launch_server, ) +# 60 test classes testing hybrid parallelism configurations +# Each test launches server + runs MMLU eval (~90s per test) +register_cuda_ci(est_time=5400, suite="weekly-8-gpu-h200", nightly=True) + class Test00(CustomTestCase): @classmethod