From 5f72d36d40d32d0d20a6a2e9ac45c045ec653b41 Mon Sep 17 00:00:00 2001
From: alisonshao <54658187+alisonshao@users.noreply.github.com>
Date: Thu, 13 Nov 2025 19:10:35 -0800
Subject: [PATCH] Fix nightly tests to fail properly when any job fails
 (#13096)

---
 .github/workflows/nightly-test-amd.yml  | 18 +++++++++++++
 .github/workflows/nightly-test-b200.yml | 22 ++++++++++++++--
 .github/workflows/nightly-test.yml      | 35 +++++++++++++++++++------
 3 files changed, 65 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/nightly-test-amd.yml b/.github/workflows/nightly-test-amd.yml
index 096e876de..932aafe8c 100644
--- a/.github/workflows/nightly-test-amd.yml
+++ b/.github/workflows/nightly-test-amd.yml
@@ -39,3 +39,21 @@ jobs:
         run: |
           bash scripts/ci/amd_ci_exec.sh -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" python3 run_suite.py --suite nightly-amd --timeout-per-file 7200
           echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY
+
+  check-all-jobs:
+    if: always() && (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch')
+    needs:
+      - nightly-test
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check if any job failed
+        run: |
+          if [[ "${{ contains(needs.*.result, 'failure') }}" == "true" ]]; then
+            echo "One or more nightly test jobs failed"
+            exit 1
+          fi
+          if [[ "${{ contains(needs.*.result, 'cancelled') }}" == "true" ]]; then
+            echo "One or more nightly test jobs were cancelled"
+            exit 1
+          fi
+          echo "All nightly test jobs passed"
diff --git a/.github/workflows/nightly-test-b200.yml b/.github/workflows/nightly-test-b200.yml
index 020163f09..4a5ccd4e3 100644
--- a/.github/workflows/nightly-test-b200.yml
+++ b/.github/workflows/nightly-test-b200.yml
@@ -18,7 +18,7 @@ jobs:
   nightly-test-4-gpu-b200:
     if: github.repository == 'sgl-project/sglang'
     runs-on: 4-gpu-b200
-    continue-on-error: true
+
     env:
       RUNNER_LABELS: 4-gpu-b200
     steps:
@@ -50,7 +50,6 @@ jobs:
   nightly-test-8-gpu-b200:
     if: github.repository == 'sgl-project/sglang'
     runs-on: 8-gpu-b200
-    continue-on-error: true
     env:
       RUNNER_LABELS: 8-gpu-b200
     steps:
@@ -96,3 +95,22 @@ jobs:
           GITHUB_RUN_NUMBER: ${{ github.run_number }}
         run: |
           python3 scripts/ci/publish_traces.py --traces-dir test/srt/performance_profiles_deepseek_v32
+
+  check-all-jobs:
+    if: always() && (github.repository == 'sgl-project/sglang' || github.event_name == 'workflow_dispatch')
+    needs:
+      - nightly-test-4-gpu-b200
+      - nightly-test-8-gpu-b200
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check if any job failed
+        run: |
+          if [[ "${{ contains(needs.*.result, 'failure') }}" == "true" ]]; then
+            echo "One or more nightly test jobs failed"
+            exit 1
+          fi
+          if [[ "${{ contains(needs.*.result, 'cancelled') }}" == "true" ]]; then
+            echo "One or more nightly test jobs were cancelled"
+            exit 1
+          fi
+          echo "All nightly test jobs passed"
diff --git a/.github/workflows/nightly-test.yml b/.github/workflows/nightly-test.yml
index f7dda4715..ed3c8e624 100644
--- a/.github/workflows/nightly-test.yml
+++ b/.github/workflows/nightly-test.yml
@@ -18,7 +18,6 @@ jobs:
   nightly-test-eval-text-models:
     if: github.repository == 'sgl-project/sglang'
     runs-on: 2-gpu-runner
-    continue-on-error: true
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
@@ -36,7 +35,6 @@ jobs:
   nightly-test-perf-text-models:
     if: github.repository == 'sgl-project/sglang'
     runs-on: 2-gpu-runner
-    continue-on-error: true
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
@@ -66,7 +64,6 @@ jobs:
   nightly-test-eval-vlms:
     if: github.repository == 'sgl-project/sglang'
     runs-on: 2-gpu-runner
-    continue-on-error: true
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
@@ -84,7 +81,6 @@ jobs:
   nightly-test-perf-vlms:
     if: github.repository == 'sgl-project/sglang'
     runs-on: 2-gpu-runner
-    continue-on-error: true
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
@@ -114,7 +110,7 @@ jobs:
   nightly-test-1-gpu:
     if: github.repository == 'sgl-project/sglang'
     runs-on: 1-gpu-runner
-    continue-on-error: true
+
     env:
       RUNNER_LABELS: 1-gpu-runner
     steps:
@@ -134,7 +130,6 @@ jobs:
   nightly-test-4-gpu:
     if: github.repository == 'sgl-project/sglang'
     runs-on: 4-gpu-h100
-    continue-on-error: true
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
@@ -152,7 +147,6 @@ jobs:
   nightly-test-8-gpu-h200:
     if: github.repository == 'sgl-project/sglang'
     runs-on: 8-gpu-h200
-    continue-on-error: true
     env:
       RUNNER_LABELS: 8-gpu-h200
     steps:
@@ -172,7 +166,6 @@ jobs:
   nightly-test-8-gpu-h20:
     if: github.repository == 'sgl-project/sglang'
     runs-on: 8-gpu-h20
-    continue-on-error: true
     env:
       SGLANG_CI_RDMA_ALL_DEVICES: "mlx5_1,mlx5_2,mlx5_3,mlx5_4"
     steps:
@@ -188,3 +181,29 @@ jobs:
         run: |
           cd test/srt
           python3 run_suite.py --suite nightly-8-gpu-h20 --continue-on-error
+
+  check-all-jobs:
+    if: always() && (github.repository == 'sgl-project/sglang' || github.event_name == 'workflow_dispatch')
+    needs:
+      - nightly-test-eval-text-models
+      - nightly-test-perf-text-models
+      - nightly-test-eval-vlms
+      - nightly-test-perf-vlms
+      - nightly-test-1-gpu
+      - nightly-test-4-gpu
+      - nightly-test-8-gpu-h200
+      - nightly-test-8-gpu-h20
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check if any job failed
+        run: |
+          # Now that continue-on-error is removed, failures will be properly reported
+          if [[ "${{ contains(needs.*.result, 'failure') }}" == "true" ]]; then
+            echo "One or more nightly test jobs failed"
+            exit 1
+          fi
+          if [[ "${{ contains(needs.*.result, 'cancelled') }}" == "true" ]]; then
+            echo "One or more nightly test jobs were cancelled"
+            exit 1
+          fi
+          echo "All nightly test jobs passed"