Add model validation for all GPU runners to prevent cache corruption (#13171)

Co-authored-by: Kangyan Zhou <zky314343421@gmail.com>

.github/workflows/nightly-test-b200.yml

@@ -19,6 +19,8 @@ jobs:
     if: github.repository == 'sgl-project/sglang'
     runs-on: 4-gpu-b200
     continue-on-error: true
+    env:
+      RUNNER_LABELS: 4-gpu-b200
     steps:
       - name: Checkout code
         uses: actions/checkout@v4

.github/workflows/nightly-test.yml

@@ -115,6 +115,8 @@ jobs:
     if: github.repository == 'sgl-project/sglang'
     runs-on: 1-gpu-runner
     continue-on-error: true
+    env:
+      RUNNER_LABELS: 1-gpu-runner
     steps:
       - name: Checkout code
         uses: actions/checkout@v4

.github/workflows/pr-test-amd.yml

@@ -192,6 +192,8 @@ jobs:
     needs: [check-changes, unit-test-backend-2-gpu-amd]
     if: always() && !failure() && !cancelled() &&
       ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
+    env:
+      RUNNER_LABELS: linux-mi300-gpu-8
     strategy:
       fail-fast: false
       matrix:

.github/workflows/pr-test.yml

@@ -202,6 +202,8 @@ jobs:
     needs: [check-changes, sgl-kernel-build-wheels]
     if: needs.check-changes.outputs.sgl_kernel == 'true'
     runs-on: 1-gpu-runner
+    env:
+      RUNNER_LABELS: 1-gpu-runner
     steps:
       - uses: actions/checkout@v4
 
@@ -231,6 +233,8 @@ jobs:
     needs: [check-changes, sgl-kernel-build-wheels]
     if: needs.check-changes.outputs.sgl_kernel == 'true'
     runs-on: 1-gpu-runner
+    env:
+      RUNNER_LABELS: 1-gpu-runner
     steps:
       - uses: actions/checkout@v4
 
@@ -262,6 +266,7 @@ jobs:
     runs-on: 1-gpu-runner
     env:
       CI: true
+      RUNNER_LABELS: 1-gpu-runner
     steps:
       - uses: actions/checkout@v4
 
@@ -350,6 +355,8 @@ jobs:
     if: always() && !failure() && !cancelled() &&
       ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
     runs-on: 1-gpu-runner
+    env:
+      RUNNER_LABELS: 1-gpu-runner
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
@@ -377,6 +384,8 @@ jobs:
     if: always() && !failure() && !cancelled() &&
       ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
     runs-on: 1-gpu-runner
+    env:
+      RUNNER_LABELS: 1-gpu-runner
     strategy:
       fail-fast: false
       max-parallel: 5
@@ -409,6 +418,8 @@ jobs:
     if: always() && !failure() && !cancelled() &&
       ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
     runs-on: 2-gpu-runner
+    env:
+      RUNNER_LABELS: 2-gpu-runner
     strategy:
       fail-fast: false
       matrix:
@@ -440,6 +451,8 @@ jobs:
     if: always() && !failure() && !cancelled() &&
       ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
     runs-on: 4-gpu-h100
+    env:
+      RUNNER_LABELS: 4-gpu-h100
     strategy:
       fail-fast: false
       matrix:
@@ -506,6 +519,7 @@ jobs:
     runs-on: 8-gpu-h20
     env:
       SGLANG_CI_RDMA_ALL_DEVICES: "mlx5_1,mlx5_2,mlx5_3,mlx5_4"
+      RUNNER_LABELS: 8-gpu-h20
     strategy:
       fail-fast: false
       matrix:
@@ -537,6 +551,8 @@ jobs:
     if: always() && !failure() && !cancelled() &&
       ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
     runs-on: 1-gpu-runner
+    env:
+      RUNNER_LABELS: 1-gpu-runner
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
@@ -596,6 +612,8 @@ jobs:
     if: always() && !failure() && !cancelled() &&
       ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
     runs-on: 1-gpu-runner
+    env:
+      RUNNER_LABELS: 1-gpu-runner
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
@@ -647,6 +665,8 @@ jobs:
     if: always() && !failure() && !cancelled() &&
       ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
     runs-on: 1-gpu-runner
+    env:
+      RUNNER_LABELS: 1-gpu-runner
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
@@ -680,6 +700,8 @@ jobs:
     if: always() && !failure() && !cancelled() &&
       ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
     runs-on: 2-gpu-runner
+    env:
+      RUNNER_LABELS: 2-gpu-runner
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
@@ -737,6 +759,8 @@ jobs:
     if: always() && !failure() && !cancelled() &&
       ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
     runs-on: 1-gpu-runner
+    env:
+      RUNNER_LABELS: 1-gpu-runner
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
@@ -767,6 +791,8 @@ jobs:
     if: always() && !failure() && !cancelled() &&
       ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
     runs-on: 2-gpu-runner
+    env:
+      RUNNER_LABELS: 2-gpu-runner
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
@@ -797,6 +823,8 @@ jobs:
     if: always() && !failure() && !cancelled() &&
       ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
     runs-on: 4-gpu-h100
+    env:
+      RUNNER_LABELS: 4-gpu-h100
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
@@ -853,6 +881,8 @@ jobs:
     if: always() && !failure() && !cancelled() &&
       ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
     runs-on: 4-gpu-b200
+    env:
+      RUNNER_LABELS: 4-gpu-b200
     strategy:
       fail-fast: false
     steps:
@@ -882,6 +912,8 @@ jobs:
     if: always() && !failure() && !cancelled() &&
       ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
     runs-on: 4-gpu-gb200
+    env:
+      RUNNER_LABELS: 4-gpu-gb200
     strategy:
       fail-fast: false
     steps:

scripts/ci/validate_and_download_models.py

@@ -37,7 +37,39 @@ except ImportError:
 # Mapping of runner labels to their required models
 # Add new runner labels and models here as needed
 RUNNER_LABEL_MODEL_MAP: Dict[str, List[str]] = {
-    "8-gpu-h200": ["deepseek-ai/DeepSeek-V3-0324", "moonshotai/Kimi-K2-Thinking"],
+    "1-gpu-runner": [
+        "deepseek-ai/DeepSeek-OCR",
+        "google/gemma-3-4b-it",
+        "lmms-lab/llava-onevision-qwen2-0.5b-ov",
+        "lmsys/sglang-ci-dsv3-test",
+        "lmsys/sglang-EAGLE-llama2-chat-7B",
+        "lmsys/sglang-EAGLE3-LLaMA3.1-Instruct-8B",
+        "meta-llama/Llama-2-7b-chat-hf",
+        "meta-llama/Llama-3.2-1B-Instruct",
+        "meta-llama/Llama-3.1-8B-Instruct",
+        "mistralai/Mixtral-8x7B-Instruct-v0.1",
+        "moonshotai/Kimi-VL-A3B-Instruct",
+        "nvidia/NVIDIA-Nemotron-Nano-9B-v2",
+        "nvidia/NVIDIA-Nemotron-Nano-9B-v2-FP8",
+        "OpenGVLab/InternVL2_5-2B",
+        "Qwen/Qwen2.5-7B-Instruct",
+        "Qwen/Qwen3-8B",
+        "Qwen/Qwen3-Coder-30B-A3B-Instruct",
+        "Qwen/QwQ-32B-AWQ",
+    ],
+    "2-gpu-runner": [
+        "mistralai/Mixtral-8x7B-Instruct-v0.1",
+        "moonshotai/Kimi-Linear-48B-A3B-Instruct",
+    ],
+    "8-gpu-h200": [
+        "deepseek-ai/DeepSeek-V3-0324",
+        "deepseek-ai/DeepSeek-V3.2-Exp",
+        "moonshotai/Kimi-K2-Thinking",
+    ],
+    "8-gpu-b200": ["deepseek-ai/DeepSeek-V3.1", "deepseek-ai/DeepSeek-V3.2-Exp"],
+    "4-gpu-b200": ["nvidia/DeepSeek-V3-0324-FP4"],
+    "4-gpu-gb200": ["nvidia/DeepSeek-V3-0324-FP4"],
+    "4-gpu-h100": ["lmsys/sglang-ci-dsv3-test", "lmsys/sglang-ci-dsv3-test-NextN"],
 }