diff --git a/.github/workflows/pr-test.yml b/.github/workflows/pr-test.yml index 74446bd1e..d621053da 100644 --- a/.github/workflows/pr-test.yml +++ b/.github/workflows/pr-test.yml @@ -609,6 +609,48 @@ jobs: cd test/srt python3 run_suite.py --suite per-commit-1-gpu --auto-partition-id ${{ matrix.part }} --auto-partition-size 15 + + stage-b-test-4-gpu-b200: + needs: [check-changes, call-gate, unit-test-backend-1-gpu] + if: | + always() && + ( + (inputs.target_stage == 'stage-b-test-4-gpu-b200') || + ( + !inputs.target_stage && + (github.event_name == 'schedule' || (!failure() && !cancelled())) && + ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true')) + ) + ) + runs-on: ${{ needs.check-changes.outputs.b200_runner }} + env: + RUNNER_LABELS: ${{ needs.check-changes.outputs.b200_runner }} + strategy: + fail-fast: false + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Download artifacts + if: needs.check-changes.outputs.sgl_kernel == 'true' + uses: actions/download-artifact@v6 + with: + path: sgl-kernel/dist/ + merge-multiple: true + pattern: wheel-python3.10-cuda12.9 + + - name: Install dependencies + run: | + CUSTOM_BUILD_SGL_KERNEL=${{needs.check-changes.outputs.sgl_kernel}} IS_BLACKWELL=1 bash scripts/ci/ci_install_dependency.sh + + - name: Run test + timeout-minutes: 30 + run: | + cd test/srt + IS_BLACKWELL=1 python3 run_suite.py --suite per-commit-4-gpu-b200-stage-b --timeout-per-file 1800 + + unit-test-backend-2-gpu: needs: [check-changes, call-gate, unit-test-backend-1-gpu] if: | @@ -651,7 +693,7 @@ jobs: python3 run_suite.py --suite per-commit-2-gpu --auto-partition-id ${{ matrix.part }} --auto-partition-size 2 unit-test-backend-4-gpu: - needs: [check-changes, call-gate, unit-test-backend-2-gpu] + needs: [check-changes, call-gate, unit-test-backend-2-gpu, stage-b-test-4-gpu-b200] if: | always() && ( @@ -692,7 +734,7 @@ jobs: python3 run_suite.py --suite per-commit-4-gpu --auto-partition-id ${{ matrix.part }} --auto-partition-size 3 unit-test-backend-8-gpu-h200: - needs: [check-changes, call-gate, unit-test-backend-2-gpu] + needs: [check-changes, call-gate, unit-test-backend-2-gpu, stage-b-test-4-gpu-b200] if: | always() && ( @@ -739,7 +781,7 @@ jobs: python3 run_suite.py --suite per-commit-8-gpu-h200 --auto-partition-id ${{ matrix.part }} --auto-partition-size 3 unit-test-backend-8-gpu-h20: - needs: [check-changes, call-gate, unit-test-backend-2-gpu] + needs: [check-changes, call-gate, unit-test-backend-2-gpu, stage-b-test-4-gpu-b200] if: | always() && ( @@ -966,7 +1008,7 @@ jobs: python3 -m unittest test_bench_serving.TestBenchServing.test_embeddings_api_batch_scaling performance-test-2-gpu: - needs: [check-changes, call-gate, unit-test-backend-2-gpu] + needs: [check-changes, call-gate, unit-test-backend-2-gpu, stage-b-test-4-gpu-b200] if: | always() && ( @@ -1113,7 +1155,7 @@ jobs: python3 test_moe_eval_accuracy_large.py unit-test-deepep-4-gpu: - needs: [check-changes, call-gate, unit-test-backend-2-gpu] + needs: [check-changes, call-gate, unit-test-backend-2-gpu, stage-b-test-4-gpu-b200] if: | always() && ( @@ -1150,7 +1192,7 @@ jobs: python3 run_suite.py --suite per-commit-4-gpu-deepep unit-test-deepep-8-gpu: - needs: [check-changes, call-gate, unit-test-backend-2-gpu] + needs: [check-changes, call-gate, unit-test-backend-2-gpu, stage-b-test-4-gpu-b200] if: | always() && ( @@ -1187,7 +1229,7 @@ jobs: python3 run_suite.py --suite per-commit-8-gpu-h200-deepep unit-test-backend-4-gpu-b200: - needs: [check-changes, call-gate, unit-test-backend-2-gpu] + needs: [check-changes, call-gate, unit-test-backend-2-gpu, stage-b-test-4-gpu-b200] if: | always() && ( @@ -1229,7 +1271,7 @@ jobs: IS_BLACKWELL=1 python3 run_suite.py --suite per-commit-4-gpu-b200 --auto-partition-id ${{ matrix.part }} --auto-partition-size 3 --timeout-per-file 1800 unit-test-backend-4-gpu-gb200: - needs: [check-changes, call-gate, unit-test-backend-2-gpu, sgl-kernel-build-wheels-arm] + needs: [check-changes, call-gate, unit-test-backend-2-gpu, stage-b-test-4-gpu-b200, sgl-kernel-build-wheels-arm] if: | always() && ( @@ -1287,6 +1329,7 @@ jobs: quantization-test, unit-test-backend-1-gpu, unit-test-backend-2-gpu, + stage-b-test-4-gpu-b200, unit-test-backend-4-gpu, unit-test-backend-8-gpu-h20, unit-test-backend-8-gpu-h200, diff --git a/scripts/ci/slash_command_handler.py b/scripts/ci/slash_command_handler.py index bb09d52a5..fa3bb8557 100644 --- a/scripts/ci/slash_command_handler.py +++ b/scripts/ci/slash_command_handler.py @@ -149,6 +149,7 @@ def handle_rerun_stage( "quantization-test", "unit-test-backend-1-gpu", "unit-test-backend-2-gpu", + "stage-b-test-4-gpu-b200", "unit-test-backend-4-gpu", "unit-test-backend-8-gpu-h200", "unit-test-backend-8-gpu-h20", diff --git a/scripts/ci_monitor/ci_analyzer.py b/scripts/ci_monitor/ci_analyzer.py index 05bbab28c..409960f16 100755 --- a/scripts/ci_monitor/ci_analyzer.py +++ b/scripts/ci_monitor/ci_analyzer.py @@ -105,6 +105,7 @@ class SGLangCIAnalyzer: "stage-a-test-1", "unit-test-backend-1-gpu", "unit-test-backend-2-gpu", + "stage-b-test-4-gpu-b200", "unit-test-backend-4-gpu", "unit-test-backend-8-gpu", ], @@ -213,6 +214,7 @@ class SGLangCIAnalyzer: "stage-a-test-1", "unit-test-backend-1-gpu", "unit-test-backend-2-gpu", + "stage-b-test-4-gpu-b200", "unit-test-backend-4-gpu", "unit-test-backend-8-gpu-h200", "unit-test-backend-8-gpu-h20", diff --git a/scripts/ci_monitor/ci_analyzer_balance.py b/scripts/ci_monitor/ci_analyzer_balance.py index e0779990c..9217b3c81 100755 --- a/scripts/ci_monitor/ci_analyzer_balance.py +++ b/scripts/ci_monitor/ci_analyzer_balance.py @@ -177,6 +177,7 @@ class SGLangTestBalanceAnalyzer: "stage-a-test-1", "unit-test-backend-1-gpu", "unit-test-backend-2-gpu", + "stage-b-test-4-gpu-b200", "unit-test-backend-4-gpu", "unit-test-backend-8-gpu-h200", "unit-test-backend-8-gpu-h20", diff --git a/test/srt/run_suite.py b/test/srt/run_suite.py index 52389e8c2..ca84c0612 100644 --- a/test/srt/run_suite.py +++ b/test/srt/run_suite.py @@ -167,8 +167,10 @@ suites = { TestFile("test_disaggregation_pp.py", 140), TestFile("test_disaggregation_dp_attention.py", 155), ], + "per-commit-4-gpu-b200-stage-b": [ + TestFile("test_deepseek_v3_fp4_4gpu.py", 1800), # Stage B test + ], "per-commit-4-gpu-b200": [ - TestFile("test_deepseek_v3_fp4_4gpu.py", 1800), TestFile("test_flash_attention_4.py", 90), TestFile("test_fp8_blockwise_gemm.py", 280), TestFile("test_gpt_oss_4gpu.py", 700),