diff --git a/.github/workflows/nightly-test-nvidia.yml b/.github/workflows/nightly-test-nvidia.yml index b600d7077..8190c7e1d 100644 --- a/.github/workflows/nightly-test-nvidia.yml +++ b/.github/workflows/nightly-test-nvidia.yml @@ -98,6 +98,10 @@ jobs: nightly-test-general-8-gpu-h200: if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-general-8-gpu-h200') runs-on: 8-gpu-h200 + strategy: + fail-fast: false + matrix: + partition: [0, 1, 2] env: RUNNER_LABELS: 8-gpu-h200 steps: @@ -120,7 +124,7 @@ jobs: IS_H200: "1" run: | cd test - python3 run_suite.py --hw cuda --suite nightly-8-gpu-common --nightly --timeout-per-file=18000 --continue-on-error + python3 run_suite.py --hw cuda --suite nightly-8-gpu-common --nightly --timeout-per-file=18000 --continue-on-error --auto-partition-id=${{ matrix.partition }} --auto-partition-size=3 - name: Run test timeout-minutes: 30 @@ -156,8 +160,12 @@ jobs: # General tests - 8 GPU B200 nightly-test-general-8-gpu-b200: - if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-general-8-gpu-h20') + if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-general-8-gpu-b200') runs-on: 8-gpu-b200 + strategy: + fail-fast: false + matrix: + partition: [0, 1, 2] steps: - name: Checkout code uses: actions/checkout@v4 @@ -177,7 +185,7 @@ jobs: GPU_CONFIG: "8-gpu-b200" run: | cd test - IS_BLACKWELL=1 python3 run_suite.py --hw cuda --suite nightly-8-gpu-common --nightly --timeout-per-file=12000 --continue-on-error + IS_BLACKWELL=1 python3 run_suite.py --hw cuda --suite nightly-8-gpu-common --nightly --timeout-per-file=12000 --continue-on-error --auto-partition-id=${{ matrix.partition }} --auto-partition-size=3 # Text model accuracy tests nightly-test-text-accuracy-2-gpu-runner: diff --git a/test/registered/8-gpu-models/test_deepseek_v31.py b/test/registered/8-gpu-models/test_deepseek_v31.py index 7f381a0bc..79c4a2bfb 100644 --- a/test/registered/8-gpu-models/test_deepseek_v31.py +++ b/test/registered/8-gpu-models/test_deepseek_v31.py @@ -7,7 +7,7 @@ from sglang.test.run_combined_tests import run_combined_tests from sglang.test.test_utils import ModelLaunchSettings # Runs on both H200 and B200 via nightly-8-gpu-common suite -register_cuda_ci(est_time=12000, suite="nightly-8-gpu-common", nightly=True) +register_cuda_ci(est_time=5400, suite="nightly-8-gpu-common", nightly=True) DEEPSEEK_V31_MODEL_PATH = "deepseek-ai/DeepSeek-V3.1" diff --git a/test/registered/8-gpu-models/test_deepseek_v32.py b/test/registered/8-gpu-models/test_deepseek_v32.py index bd46c51eb..1abf4d3fa 100644 --- a/test/registered/8-gpu-models/test_deepseek_v32.py +++ b/test/registered/8-gpu-models/test_deepseek_v32.py @@ -6,7 +6,7 @@ from sglang.test.performance_test_runner import PerformanceTestParams from sglang.test.run_combined_tests import run_combined_tests from sglang.test.test_utils import ModelLaunchSettings, is_blackwell_system -register_cuda_ci(est_time=18000, suite="nightly-8-gpu-common", nightly=True) +register_cuda_ci(est_time=5400, suite="nightly-8-gpu-common", nightly=True) DEEPSEEK_V32_MODEL_PATH = "deepseek-ai/DeepSeek-V3.2" diff --git a/test/registered/8-gpu-models/test_deepseek_v32_cp_single_node.py b/test/registered/8-gpu-models/test_deepseek_v32_cp_single_node.py index b704e7618..d4f00a42e 100644 --- a/test/registered/8-gpu-models/test_deepseek_v32_cp_single_node.py +++ b/test/registered/8-gpu-models/test_deepseek_v32_cp_single_node.py @@ -5,7 +5,7 @@ from sglang.test.ci.ci_register import register_cuda_ci from sglang.test.run_combined_tests import run_combined_tests from sglang.test.test_utils import ModelLaunchSettings, is_blackwell_system -register_cuda_ci(est_time=18000, suite="nightly-8-gpu-common", nightly=True) +register_cuda_ci(est_time=5400, suite="nightly-8-gpu-common", nightly=True) DEEPSEEK_V32_EXP_MODEL_PATH = "deepseek-ai/DeepSeek-V3.2-Exp" diff --git a/test/registered/8-gpu-models/test_glm_46.py b/test/registered/8-gpu-models/test_glm_46.py index 1a9e17a11..d6e4a7bd8 100644 --- a/test/registered/8-gpu-models/test_glm_46.py +++ b/test/registered/8-gpu-models/test_glm_46.py @@ -7,7 +7,7 @@ from sglang.test.run_combined_tests import run_combined_tests from sglang.test.test_utils import ModelLaunchSettings # Runs on both H200 and B200 via nightly-8-gpu-common suite -register_cuda_ci(est_time=12000, suite="nightly-8-gpu-common", nightly=True) +register_cuda_ci(est_time=1800, suite="nightly-8-gpu-common", nightly=True) GLM_4_6_MODEL_PATH = "zai-org/GLM-4.6" diff --git a/test/registered/8-gpu-models/test_glm_46_fp8.py b/test/registered/8-gpu-models/test_glm_46_fp8.py index 6c422c31c..9539a7521 100644 --- a/test/registered/8-gpu-models/test_glm_46_fp8.py +++ b/test/registered/8-gpu-models/test_glm_46_fp8.py @@ -7,7 +7,7 @@ from sglang.test.run_combined_tests import run_combined_tests from sglang.test.test_utils import ModelLaunchSettings # Runs on both H200 and B200 via nightly-8-gpu-common suite -register_cuda_ci(est_time=12000, suite="nightly-8-gpu-common", nightly=True) +register_cuda_ci(est_time=1800, suite="nightly-8-gpu-common", nightly=True) GLM_4_6_FP8_MODEL_PATH = "zai-org/GLM-4.6-FP8" diff --git a/test/registered/8-gpu-models/test_kimi_k2.py b/test/registered/8-gpu-models/test_kimi_k2.py index c55ac896a..e8ec07f8c 100644 --- a/test/registered/8-gpu-models/test_kimi_k2.py +++ b/test/registered/8-gpu-models/test_kimi_k2.py @@ -7,7 +7,7 @@ from sglang.test.run_combined_tests import run_combined_tests from sglang.test.test_utils import ModelLaunchSettings # Runs on both H200 and B200 via nightly-8-gpu-common suite -register_cuda_ci(est_time=12000, suite="nightly-8-gpu-common", nightly=True) +register_cuda_ci(est_time=1800, suite="nightly-8-gpu-common", nightly=True) KIMI_K2_THINKING_MODEL_PATH = "moonshotai/Kimi-K2-Thinking" diff --git a/test/registered/8-gpu-models/test_llama4.py b/test/registered/8-gpu-models/test_llama4.py index 12b3414ab..c4d362184 100644 --- a/test/registered/8-gpu-models/test_llama4.py +++ b/test/registered/8-gpu-models/test_llama4.py @@ -7,7 +7,7 @@ from sglang.test.run_combined_tests import run_combined_tests from sglang.test.test_utils import ModelLaunchSettings # Runs on both H200 and B200 via nightly-8-gpu-common suite -register_cuda_ci(est_time=12000, suite="nightly-8-gpu-common", nightly=True) +register_cuda_ci(est_time=1800, suite="nightly-8-gpu-common", nightly=True) LLAMA4_MODEL_PATH = "meta-llama/Llama-4-Scout-17B-16E-Instruct" diff --git a/test/registered/8-gpu-models/test_minimax_m2.py b/test/registered/8-gpu-models/test_minimax_m2.py index cf6f7ffea..d1801b253 100644 --- a/test/registered/8-gpu-models/test_minimax_m2.py +++ b/test/registered/8-gpu-models/test_minimax_m2.py @@ -7,7 +7,7 @@ from sglang.test.run_combined_tests import run_combined_tests from sglang.test.test_utils import ModelLaunchSettings # Runs on both H200 and B200 via nightly-8-gpu-common suite -register_cuda_ci(est_time=12000, suite="nightly-8-gpu-common", nightly=True) +register_cuda_ci(est_time=1800, suite="nightly-8-gpu-common", nightly=True) MINIMAX_M2_MODEL_PATH = "MiniMaxAI/MiniMax-M2" diff --git a/test/registered/8-gpu-models/test_mistral_large3.py b/test/registered/8-gpu-models/test_mistral_large3.py index 7903a915e..b795022b8 100644 --- a/test/registered/8-gpu-models/test_mistral_large3.py +++ b/test/registered/8-gpu-models/test_mistral_large3.py @@ -9,7 +9,7 @@ from sglang.test.test_utils import ModelLaunchSettings, is_blackwell_system # Runs on both H200 and B200 via nightly-8-gpu-common suite # Note: trtllm_mla backend may have hardware-specific behavior -register_cuda_ci(est_time=12000, suite="nightly-8-gpu-common", nightly=True) +register_cuda_ci(est_time=1800, suite="nightly-8-gpu-common", nightly=True) MISTRAL_LARGE3_MODEL_PATH = "mistralai/Mistral-Large-3-675B-Instruct-2512" MISTRAL_LARGE3_EAGLE_MODEL_PATH = "mistralai/Mistral-Large-3-675B-Instruct-2512-Eagle" diff --git a/test/registered/8-gpu-models/test_qwen3_235b.py b/test/registered/8-gpu-models/test_qwen3_235b.py index 3b7a32ba4..63281cd37 100644 --- a/test/registered/8-gpu-models/test_qwen3_235b.py +++ b/test/registered/8-gpu-models/test_qwen3_235b.py @@ -7,7 +7,7 @@ from sglang.test.run_combined_tests import run_combined_tests from sglang.test.test_utils import ModelLaunchSettings, is_blackwell_system # Runs on both H200 and B200 via nightly-8-gpu-common suite -register_cuda_ci(est_time=12000, suite="nightly-8-gpu-common", nightly=True) +register_cuda_ci(est_time=1800, suite="nightly-8-gpu-common", nightly=True) QWEN3_235B_MODEL_PATH = "Qwen/Qwen3-235B-A22B-Instruct-2507"