Upload nightly test metrics to GH artifacts (#17696)

.github/workflows/nightly-test-nvidia.yml

@@ -134,6 +134,26 @@ jobs:
           cd test
           python3 run_suite.py --hw cuda --suite nightly-8-gpu-h200 --nightly --continue-on-error
 
+      - name: Collect performance metrics
+        if: always()
+        run: |
+          python3 scripts/ci/save_metrics.py \
+            --gpu-config 8-gpu-h200 \
+            --partition ${{ matrix.partition }} \
+            --run-id ${{ github.run_id }} \
+            --output test/metrics-8gpu-h200-partition-${{ matrix.partition }}.json \
+            --search-dir test/performance_profiles_8_gpu \
+            --search-dir test
+
+      - name: Upload partition metrics
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: metrics-8gpu-h200-partition-${{ matrix.partition }}
+          path: test/metrics-8gpu-h200-partition-${{ matrix.partition }}.json
+          retention-days: 5
+          if-no-files-found: ignore
+
   # General tests - 8 GPU H20
   nightly-test-general-8-gpu-h20:
     if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-general-8-gpu-h20')
@@ -187,6 +207,26 @@ jobs:
           cd test
           IS_BLACKWELL=1 python3 run_suite.py --hw cuda --suite nightly-8-gpu-common --nightly --timeout-per-file=12000 --continue-on-error --auto-partition-id=${{ matrix.partition }} --auto-partition-size=3
 
+      - name: Collect performance metrics
+        if: always()
+        run: |
+          python3 scripts/ci/save_metrics.py \
+            --gpu-config 8-gpu-b200 \
+            --partition ${{ matrix.partition }} \
+            --run-id ${{ github.run_id }} \
+            --output test/metrics-8gpu-b200-partition-${{ matrix.partition }}.json \
+            --search-dir test/performance_profiles_8_gpu \
+            --search-dir test
+
+      - name: Upload partition metrics
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: metrics-8gpu-b200-partition-${{ matrix.partition }}
+          path: test/metrics-8gpu-b200-partition-${{ matrix.partition }}.json
+          retention-days: 5
+          if-no-files-found: ignore
+
   # Text model accuracy tests
   nightly-test-text-accuracy-2-gpu-runner:
     if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-text-accuracy-2-gpu-runner')
@@ -403,6 +443,48 @@ jobs:
           cd test
           python3 run_suite.py --hw cuda --suite nightly-8-gpu-b200 --nightly --continue-on-error --timeout-per-file 2400
 
+  # Consolidate performance metrics from all 8-GPU jobs
+  consolidate-metrics:
+    if: github.repository == 'sgl-project/sglang' && always()
+    needs:
+      - nightly-test-general-8-gpu-h200
+      - nightly-test-general-8-gpu-b200
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.ref || github.ref }}
+
+      - name: Download all partition metrics
+        uses: actions/download-artifact@v4
+        with:
+          pattern: metrics-*
+          path: metrics/
+          merge-multiple: true
+
+      - name: List downloaded metrics
+        run: |
+          echo "Downloaded metrics files:"
+          find metrics/ -name "*.json" -type f 2>/dev/null || echo "No metrics files found"
+
+      - name: Merge metrics
+        run: |
+          python3 scripts/ci/merge_metrics.py \
+            --input-dir metrics/ \
+            --output consolidated-metrics-${{ github.run_id }}.json \
+            --run-id ${{ github.run_id }} \
+            --commit-sha ${{ github.sha }} \
+            --branch ${{ github.ref_name }}
+
+      - name: Upload consolidated metrics
+        uses: actions/upload-artifact@v4
+        with:
+          name: consolidated-metrics-${{ github.run_id }}
+          path: consolidated-metrics-${{ github.run_id }}.json
+          retention-days: 90
+          if-no-files-found: warn
+
   # Final check job
   check-all-jobs:
     if: github.repository == 'sgl-project/sglang' && always()
@@ -420,6 +502,7 @@ jobs:
       - nightly-test-multimodal-server-2-gpu
       - nightly-test-perf-4-gpu-b200
       - nightly-test-specialized-8-gpu-b200
+      - consolidate-metrics
     runs-on: ubuntu-latest
     steps:
       - name: Check if any job failed