Add model validation for all GPU runners to prevent cache corruption (#13171)

Co-authored-by: Kangyan Zhou <zky314343421@gmail.com>
This commit is contained in:
alisonshao
2025-11-13 10:16:41 -08:00
committed by GitHub
parent 85b8c5c4cd
commit 9bd511a582
5 changed files with 71 additions and 1 deletions

View File

@@ -19,6 +19,8 @@ jobs:
if: github.repository == 'sgl-project/sglang'
runs-on: 4-gpu-b200
continue-on-error: true
env:
RUNNER_LABELS: 4-gpu-b200
steps:
- name: Checkout code
uses: actions/checkout@v4

View File

@@ -115,6 +115,8 @@ jobs:
if: github.repository == 'sgl-project/sglang'
runs-on: 1-gpu-runner
continue-on-error: true
env:
RUNNER_LABELS: 1-gpu-runner
steps:
- name: Checkout code
uses: actions/checkout@v4

View File

@@ -192,6 +192,8 @@ jobs:
needs: [check-changes, unit-test-backend-2-gpu-amd]
if: always() && !failure() && !cancelled() &&
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
env:
RUNNER_LABELS: linux-mi300-gpu-8
strategy:
fail-fast: false
matrix:

View File

@@ -202,6 +202,8 @@ jobs:
needs: [check-changes, sgl-kernel-build-wheels]
if: needs.check-changes.outputs.sgl_kernel == 'true'
runs-on: 1-gpu-runner
env:
RUNNER_LABELS: 1-gpu-runner
steps:
- uses: actions/checkout@v4
@@ -231,6 +233,8 @@ jobs:
needs: [check-changes, sgl-kernel-build-wheels]
if: needs.check-changes.outputs.sgl_kernel == 'true'
runs-on: 1-gpu-runner
env:
RUNNER_LABELS: 1-gpu-runner
steps:
- uses: actions/checkout@v4
@@ -262,6 +266,7 @@ jobs:
runs-on: 1-gpu-runner
env:
CI: true
RUNNER_LABELS: 1-gpu-runner
steps:
- uses: actions/checkout@v4
@@ -350,6 +355,8 @@ jobs:
if: always() && !failure() && !cancelled() &&
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
runs-on: 1-gpu-runner
env:
RUNNER_LABELS: 1-gpu-runner
steps:
- name: Checkout code
uses: actions/checkout@v4
@@ -377,6 +384,8 @@ jobs:
if: always() && !failure() && !cancelled() &&
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
runs-on: 1-gpu-runner
env:
RUNNER_LABELS: 1-gpu-runner
strategy:
fail-fast: false
max-parallel: 5
@@ -409,6 +418,8 @@ jobs:
if: always() && !failure() && !cancelled() &&
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
runs-on: 2-gpu-runner
env:
RUNNER_LABELS: 2-gpu-runner
strategy:
fail-fast: false
matrix:
@@ -440,6 +451,8 @@ jobs:
if: always() && !failure() && !cancelled() &&
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
runs-on: 4-gpu-h100
env:
RUNNER_LABELS: 4-gpu-h100
strategy:
fail-fast: false
matrix:
@@ -506,6 +519,7 @@ jobs:
runs-on: 8-gpu-h20
env:
SGLANG_CI_RDMA_ALL_DEVICES: "mlx5_1,mlx5_2,mlx5_3,mlx5_4"
RUNNER_LABELS: 8-gpu-h20
strategy:
fail-fast: false
matrix:
@@ -537,6 +551,8 @@ jobs:
if: always() && !failure() && !cancelled() &&
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
runs-on: 1-gpu-runner
env:
RUNNER_LABELS: 1-gpu-runner
steps:
- name: Checkout code
uses: actions/checkout@v4
@@ -596,6 +612,8 @@ jobs:
if: always() && !failure() && !cancelled() &&
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
runs-on: 1-gpu-runner
env:
RUNNER_LABELS: 1-gpu-runner
steps:
- name: Checkout code
uses: actions/checkout@v4
@@ -647,6 +665,8 @@ jobs:
if: always() && !failure() && !cancelled() &&
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
runs-on: 1-gpu-runner
env:
RUNNER_LABELS: 1-gpu-runner
steps:
- name: Checkout code
uses: actions/checkout@v4
@@ -680,6 +700,8 @@ jobs:
if: always() && !failure() && !cancelled() &&
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
runs-on: 2-gpu-runner
env:
RUNNER_LABELS: 2-gpu-runner
steps:
- name: Checkout code
uses: actions/checkout@v4
@@ -737,6 +759,8 @@ jobs:
if: always() && !failure() && !cancelled() &&
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
runs-on: 1-gpu-runner
env:
RUNNER_LABELS: 1-gpu-runner
steps:
- name: Checkout code
uses: actions/checkout@v4
@@ -767,6 +791,8 @@ jobs:
if: always() && !failure() && !cancelled() &&
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
runs-on: 2-gpu-runner
env:
RUNNER_LABELS: 2-gpu-runner
steps:
- name: Checkout code
uses: actions/checkout@v4
@@ -797,6 +823,8 @@ jobs:
if: always() && !failure() && !cancelled() &&
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
runs-on: 4-gpu-h100
env:
RUNNER_LABELS: 4-gpu-h100
steps:
- name: Checkout code
uses: actions/checkout@v4
@@ -853,6 +881,8 @@ jobs:
if: always() && !failure() && !cancelled() &&
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
runs-on: 4-gpu-b200
env:
RUNNER_LABELS: 4-gpu-b200
strategy:
fail-fast: false
steps:
@@ -882,6 +912,8 @@ jobs:
if: always() && !failure() && !cancelled() &&
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
runs-on: 4-gpu-gb200
env:
RUNNER_LABELS: 4-gpu-gb200
strategy:
fail-fast: false
steps: