diff --git a/.github/workflows/nightly-test-nvidia.yml b/.github/workflows/nightly-test-nvidia.yml index fc6c6e42f..b600d7077 100644 --- a/.github/workflows/nightly-test-nvidia.yml +++ b/.github/workflows/nightly-test-nvidia.yml @@ -168,13 +168,16 @@ jobs: run: | IS_BLACKWELL=1 bash scripts/ci/ci_install_dependency.sh - - name: Run test - timeout-minutes: 120 + - name: Run common 8-GPU model tests + if: always() + timeout-minutes: 300 env: + TRACE_BASE_URL: https://raw.githubusercontent.com/sglang-bot/sglang-ci-data/main/traces/${{ github.run_id }} + PERFETTO_RELAY_URL: ${{ vars.PERFETTO_RELAY_URL }} GPU_CONFIG: "8-gpu-b200" run: | cd test - python3 run_suite.py --hw cuda --suite nightly-8-gpu-b200 --nightly --continue-on-error --timeout-per-file 2400 + IS_BLACKWELL=1 python3 run_suite.py --hw cuda --suite nightly-8-gpu-common --nightly --timeout-per-file=12000 --continue-on-error # Text model accuracy tests nightly-test-text-accuracy-2-gpu-runner: @@ -368,8 +371,8 @@ jobs: cd test python3 run_suite.py --hw cuda --suite nightly-4-gpu-b200 --nightly --continue-on-error - # B200 Performance tests - 8 GPU - nightly-test-perf-8-gpu-b200: + # Specialized B200 tests - 8 GPU, for specific backends and configs + nightly-test-specialized-8-gpu-b200: if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-perf-8-gpu-b200') runs-on: 8-gpu-b200 env: @@ -384,16 +387,13 @@ jobs: run: | IS_BLACKWELL=1 bash scripts/ci/ci_install_dependency.sh - - name: Run common 8-GPU model tests - if: always() - timeout-minutes: 300 + - name: Run test + timeout-minutes: 120 env: - TRACE_BASE_URL: https://raw.githubusercontent.com/sglang-bot/sglang-ci-data/main/traces/${{ github.run_id }} - PERFETTO_RELAY_URL: ${{ vars.PERFETTO_RELAY_URL }} GPU_CONFIG: "8-gpu-b200" run: | cd test - IS_BLACKWELL=1 python3 run_suite.py --hw cuda --suite nightly-8-gpu-common --nightly --timeout-per-file=12000 --continue-on-error + python3 run_suite.py --hw cuda --suite nightly-8-gpu-b200 --nightly --continue-on-error --timeout-per-file 2400 # Final check job check-all-jobs: @@ -411,7 +411,7 @@ jobs: - nightly-test-multimodal-server-1-gpu - nightly-test-multimodal-server-2-gpu - nightly-test-perf-4-gpu-b200 - - nightly-test-perf-8-gpu-b200 + - nightly-test-specialized-8-gpu-b200 runs-on: ubuntu-latest steps: - name: Check if any job failed diff --git a/test/registered/8-gpu-models/test_deepseek_v31.py b/test/registered/8-gpu-models/test_deepseek_v31.py index 56757c88f..7f381a0bc 100644 --- a/test/registered/8-gpu-models/test_deepseek_v31.py +++ b/test/registered/8-gpu-models/test_deepseek_v31.py @@ -12,7 +12,7 @@ register_cuda_ci(est_time=12000, suite="nightly-8-gpu-common", nightly=True) DEEPSEEK_V31_MODEL_PATH = "deepseek-ai/DeepSeek-V3.1" -class TestDeepseekV31Unified(unittest.TestCase): +class TestDeepseekV31(unittest.TestCase): """Unified test class for DeepSeek-V3.1 performance and accuracy. Two variants: diff --git a/test/registered/8-gpu-models/test_deepseek_v32.py b/test/registered/8-gpu-models/test_deepseek_v32.py index dbdd90bd7..bd46c51eb 100644 --- a/test/registered/8-gpu-models/test_deepseek_v32.py +++ b/test/registered/8-gpu-models/test_deepseek_v32.py @@ -27,7 +27,7 @@ GSM8K_BASELINE = 0.935 GPQA_BASELINE = 0.835 -class TestDeepseekV32Unified(unittest.TestCase): +class TestDeepseekV32(unittest.TestCase): """Unified test class for DeepSeek V3.2 performance and accuracy. Tests multiple variants with both performance and accuracy tests: diff --git a/test/registered/8-gpu-models/test_glm_46.py b/test/registered/8-gpu-models/test_glm_46.py index 7436d6a9e..1a9e17a11 100644 --- a/test/registered/8-gpu-models/test_glm_46.py +++ b/test/registered/8-gpu-models/test_glm_46.py @@ -12,7 +12,7 @@ register_cuda_ci(est_time=12000, suite="nightly-8-gpu-common", nightly=True) GLM_4_6_MODEL_PATH = "zai-org/GLM-4.6" -class TestGLM46Unified(unittest.TestCase): +class TestGLM46(unittest.TestCase): """Unified test class for GLM-4.6 performance and accuracy. Single variant with simple TP=8 configuration. diff --git a/test/registered/8-gpu-models/test_glm_46_fp8.py b/test/registered/8-gpu-models/test_glm_46_fp8.py index 9f529b4a2..6c422c31c 100644 --- a/test/registered/8-gpu-models/test_glm_46_fp8.py +++ b/test/registered/8-gpu-models/test_glm_46_fp8.py @@ -12,7 +12,7 @@ register_cuda_ci(est_time=12000, suite="nightly-8-gpu-common", nightly=True) GLM_4_6_FP8_MODEL_PATH = "zai-org/GLM-4.6-FP8" -class TestGLM46FP8Unified(unittest.TestCase): +class TestGLM46FP8(unittest.TestCase): """Unified test class for GLM-4.6-FP8 performance and accuracy. Single variant with simple TP=8 configuration. diff --git a/test/registered/8-gpu-models/test_kimi_k2.py b/test/registered/8-gpu-models/test_kimi_k2.py index 079dbcaf3..c55ac896a 100644 --- a/test/registered/8-gpu-models/test_kimi_k2.py +++ b/test/registered/8-gpu-models/test_kimi_k2.py @@ -12,7 +12,7 @@ register_cuda_ci(est_time=12000, suite="nightly-8-gpu-common", nightly=True) KIMI_K2_THINKING_MODEL_PATH = "moonshotai/Kimi-K2-Thinking" -class TestKimiK2Unified(unittest.TestCase): +class TestKimiK2(unittest.TestCase): """Unified test class for Kimi-K2-Thinking performance and accuracy. Single variant with TP=8 + tool/reasoning parsers. diff --git a/test/registered/8-gpu-models/test_llama4.py b/test/registered/8-gpu-models/test_llama4.py index 07f14f5ce..12b3414ab 100644 --- a/test/registered/8-gpu-models/test_llama4.py +++ b/test/registered/8-gpu-models/test_llama4.py @@ -12,7 +12,8 @@ register_cuda_ci(est_time=12000, suite="nightly-8-gpu-common", nightly=True) LLAMA4_MODEL_PATH = "meta-llama/Llama-4-Scout-17B-16E-Instruct" -class TestLlama4Unified(unittest.TestCase): +@unittest.skip("Blocked: Missing HF token permission for Llama 4 model") +class TestLlama4(unittest.TestCase): """Unified test class for Llama-4-Scout performance and accuracy. Llama4 has local attention mechanism with hybrid sliding window attention. diff --git a/test/registered/8-gpu-models/test_minimax_m2.py b/test/registered/8-gpu-models/test_minimax_m2.py index 740ae108a..cf6f7ffea 100644 --- a/test/registered/8-gpu-models/test_minimax_m2.py +++ b/test/registered/8-gpu-models/test_minimax_m2.py @@ -12,7 +12,7 @@ register_cuda_ci(est_time=12000, suite="nightly-8-gpu-common", nightly=True) MINIMAX_M2_MODEL_PATH = "MiniMaxAI/MiniMax-M2" -class TestMiniMaxM2Unified(unittest.TestCase): +class TestMiniMaxM2(unittest.TestCase): """Unified test class for MiniMax-M2 performance and accuracy. Single variant with TP=8 + EP=8 configuration. diff --git a/test/registered/8-gpu-models/test_mistral_large3.py b/test/registered/8-gpu-models/test_mistral_large3.py index c55d2f06b..7903a915e 100644 --- a/test/registered/8-gpu-models/test_mistral_large3.py +++ b/test/registered/8-gpu-models/test_mistral_large3.py @@ -16,7 +16,7 @@ MISTRAL_LARGE3_EAGLE_MODEL_PATH = "mistralai/Mistral-Large-3-675B-Instruct-2512- @unittest.skipIf(not is_blackwell_system(), "Requires B200") -class TestMistralLarge3Unified(unittest.TestCase): +class TestMistralLarge3(unittest.TestCase): """Unified test class for Mistral-Large-3 performance and accuracy. Two variants: diff --git a/test/registered/8-gpu-models/test_qwen3_235b.py b/test/registered/8-gpu-models/test_qwen3_235b.py index 6110f3ec1..3b7a32ba4 100644 --- a/test/registered/8-gpu-models/test_qwen3_235b.py +++ b/test/registered/8-gpu-models/test_qwen3_235b.py @@ -13,7 +13,7 @@ QWEN3_235B_MODEL_PATH = "Qwen/Qwen3-235B-A22B-Instruct-2507" @unittest.skipIf(not is_blackwell_system(), "Requires B200") -class TestQwen3235BUnified(unittest.TestCase): +class TestQwen3235B(unittest.TestCase): """Unified test class for Qwen3-235B performance and accuracy. Single variant with simple TP=8 configuration.