ci: migrate 1-GPU model tests to test/registered/models/ (#16414)

2026-01-04 18:08:01 -08:00
parent 249c356331
commit f8411ded6e
17 changed files with 111 additions and 24 deletions
--- a/.github/workflows/pr-test-amd.yml
+++ b/.github/workflows/pr-test-amd.yml
@@ -184,6 +184,46 @@ jobs:
        run: |
          docker exec -w /sglang-checkout/test ci_sglang python3 run_suite.py --hw amd --suite stage-a-test-1

+  stage-b-test-small-1-gpu-amd:
+    needs: [check-changes, stage-a-test-1-amd]
+    if: |
+      always() &&
+      (
+        (inputs.target_stage == 'stage-b-test-small-1-gpu-amd') ||
+        (
+          !inputs.target_stage &&
+          (!failure() && !cancelled()) &&
+          ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
+        )
+      )
+    strategy:
+      fail-fast: false
+      matrix:
+        runner: [linux-mi325-gpu-1]
+        part: [0, 1, 2, 3]
+    runs-on: ${{matrix.runner}}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }}
+
+      - name: Ensure VRAM is clear
+        run: bash scripts/ensure_vram_clear.sh rocm
+
+      - name: Start CI container
+        run: bash scripts/ci/amd_ci_start_container.sh
+        env:
+          GITHUB_WORKSPACE: ${{ github.workspace }}
+
+      - name: Install dependencies
+        run: bash scripts/ci/amd_ci_install_dependency.sh
+
+      - name: Run test
+        timeout-minutes: 30
+        run: |
+          docker exec -w /sglang-checkout/test ci_sglang python3 run_suite.py --hw amd --suite stage-b-test-small-1-gpu --auto-partition-id ${{ matrix.part }} --auto-partition-size 4
+
  multimodal-gen-test-1-gpu-amd:
    needs: [check-changes]
    if: needs.check-changes.outputs.multimodal_gen == 'true'
@@ -754,7 +794,6 @@ jobs:
        run: |
          bash scripts/ci/amd_ci_exec.sh -e SGLANG_USE_AITER=0 python3 test_eval_accuracy_large.py
          bash scripts/ci/amd_ci_exec.sh python3 test_eval_fp8_accuracy.py
-          bash scripts/ci/amd_ci_exec.sh python3 models/test_qwen_models.py

  accuracy-test-2-gpu-amd:
    needs: [check-changes, accuracy-test-1-gpu-amd]
@@ -844,6 +883,7 @@ jobs:
        multimodal-gen-test-2-gpu-amd,

        stage-a-test-1-amd,
+        stage-b-test-small-1-gpu-amd,
        unit-test-backend-1-gpu-amd,
        unit-test-backend-2-gpu-amd,
        unit-test-backend-8-gpu-amd,
--- a/.github/workflows/pr-test.yml
+++ b/.github/workflows/pr-test.yml
@@ -532,7 +532,7 @@ jobs:
    strategy:
      fail-fast: false
      matrix:
-        partition: [0, 1, 2, 3, 4]
+        partition: [0, 1, 2, 3, 4, 5, 6, 7]
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
@@ -559,7 +559,7 @@ jobs:
          if [[ "${{ needs.check-changes.outputs.continue_on_error }}" == "true" ]]; then
            CONTINUE_ON_ERROR_FLAG="--continue-on-error"
          fi
-          python3 run_suite.py --hw cuda --suite stage-b-test-small-1-gpu --auto-partition-id ${{ matrix.partition }} --auto-partition-size 5 $CONTINUE_ON_ERROR_FLAG
+          python3 run_suite.py --hw cuda --suite stage-b-test-small-1-gpu --auto-partition-id ${{ matrix.partition }} --auto-partition-size 8 $CONTINUE_ON_ERROR_FLAG

  stage-b-test-large-1-gpu:
    needs: [check-changes, call-gate, stage-a-test-1, sgl-kernel-build-wheels]
@@ -891,7 +891,7 @@ jobs:
      fail-fast: false
      max-parallel: ${{ fromJson(needs.check-changes.outputs.max_parallel) }}
      matrix:
-        part: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]
+        part: [0, 1, 2, 3, 4, 5]
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
@@ -922,7 +922,7 @@ jobs:
          if [[ "${{ needs.check-changes.outputs.continue_on_error }}" == "true" ]]; then
            CONTINUE_ON_ERROR_FLAG="--continue-on-error"
          fi
-          python3 run_suite.py --suite per-commit-1-gpu --auto-partition-id ${{ matrix.part }} --auto-partition-size 13 $RETRY_FLAG $CONTINUE_ON_ERROR_FLAG
+          python3 run_suite.py --suite per-commit-1-gpu --auto-partition-id ${{ matrix.part }} --auto-partition-size 6 $RETRY_FLAG $CONTINUE_ON_ERROR_FLAG


  stage-b-test-4-gpu-b200:
--- a/docker/xeon.Dockerfile
+++ b/docker/xeon.Dockerfile
@@ -45,6 +45,7 @@ RUN source $HOME/.local/bin/env && \
    cp pyproject_cpu.toml pyproject.toml && \
    uv pip install . && \
    uv pip install torch==${VER_TORCH} torchvision==${VER_TORCHVISION} triton==${VER_TRITON} --force-reinstall && \
+    uv pip install tabulate && \
    cd ../sgl-kernel && \
    cp pyproject_cpu.toml pyproject.toml && \
    uv pip install .
--- a/docker/xpu.Dockerfile
+++ b/docker/xpu.Dockerfile
@@ -60,7 +60,7 @@ RUN --mount=type=secret,id=github_token \
    cp pyproject_xpu.toml pyproject.toml && \
    pip install . && \
    pip install xgrammar --no-deps && \
-    pip install msgspec blake3 py-cpuinfo compressed_tensors gguf partial_json_parser einops --root-user-action=ignore && \
+    pip install msgspec blake3 py-cpuinfo compressed_tensors gguf partial_json_parser einops tabulate --root-user-action=ignore && \
    conda install libsqlite=3.48.0 -y && \
    # Add environment setup commands to .bashrc again (in case it was overwritten)
    echo ". /home/sdp/miniforge3/bin/activate; conda activate py${PYTHON_VERSION}; cd /home/sdp" >> /home/sdp/.bashrc
--- a/scripts/ci/slash_command_handler.py
+++ b/scripts/ci/slash_command_handler.py
@@ -176,6 +176,7 @@ def handle_rerun_stage(
    amd_stages = [
        "sgl-kernel-unit-test-amd",
        "stage-a-test-1-amd",
+        "stage-b-test-small-1-gpu-amd",
        "unit-test-backend-1-gpu-amd",
        "unit-test-backend-2-gpu-amd",
        "unit-test-backend-8-gpu-amd",
--- a/test/registered/models/test_compressed_tensors_models.py
+++ b/test/registered/models/test_compressed_tensors_models.py
@@ -1,3 +1,9 @@
+from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
+
+# Model tests for compressed tensors (FP8)
+register_cuda_ci(est_time=42, suite="stage-b-test-small-1-gpu")
+register_amd_ci(est_time=42, suite="stage-b-test-small-1-gpu")
+
 import unittest
 from types import SimpleNamespace

--- a/test/registered/models/test_cross_encoder_models.py
+++ b/test/registered/models/test_cross_encoder_models.py
@@ -1,3 +1,9 @@
+from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
+
+# Cross encoder model tests
+register_cuda_ci(est_time=100, suite="stage-b-test-small-1-gpu")
+register_amd_ci(est_time=150, suite="stage-b-test-small-1-gpu")
+
 import multiprocessing as mp
 import random
 import unittest
--- a/test/registered/models/test_embedding_models.py
+++ b/test/registered/models/test_embedding_models.py
@@ -1,3 +1,13 @@
+from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
+
+# Embedding model tests
+register_cuda_ci(est_time=73, suite="stage-b-test-small-1-gpu")
+register_amd_ci(
+    est_time=73,
+    suite="stage-b-test-small-1-gpu",
+    disabled="see https://github.com/sgl-project/sglang/issues/11127",
+)
+
 # Copyright 2023-2024 SGLang Team
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
--- a/test/registered/models/test_encoder_embedding_models.py
+++ b/test/registered/models/test_encoder_embedding_models.py
@@ -1,3 +1,8 @@
+from sglang.test.ci.ci_register import register_cuda_ci
+
+# Encoder embedding model tests (CUDA only)
+register_cuda_ci(est_time=221, suite="stage-b-test-small-1-gpu")
+
 # Copyright 2023-2024 SGLang Team
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
--- a/test/registered/models/test_generation_models.py
+++ b/test/registered/models/test_generation_models.py
@@ -1,3 +1,8 @@
+from sglang.test.ci.ci_register import register_cuda_ci
+
+# Generation model tests (CUDA only)
+register_cuda_ci(est_time=103, suite="stage-b-test-small-1-gpu")
+
 # Copyright 2023-2024 SGLang Team
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
--- a/test/registered/models/test_nvidia_nemotron_nano_v2_vl.py
+++ b/test/registered/models/test_nvidia_nemotron_nano_v2_vl.py
@@ -1,3 +1,9 @@
+from sglang.test.ci.ci_register import register_cuda_ci
+
+# NVIDIA Nemotron Nano V2 VL model tests (CUDA only)
+# GSM8k + MMMU evaluation
+register_cuda_ci(est_time=214, suite="stage-b-test-small-1-gpu")
+
 import unittest

 from sglang.test.kits.gsm8k_accuracy_kit import GSM8KMixin
--- a/test/registered/models/test_qwen_models.py
+++ b/test/registered/models/test_qwen_models.py
@@ -1,3 +1,9 @@
+from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
+
+# Qwen model tests
+register_cuda_ci(est_time=90, suite="stage-b-test-small-1-gpu")
+register_amd_ci(est_time=82, suite="stage-b-test-small-1-gpu")
+
 import unittest
 from types import SimpleNamespace

--- a/test/registered/models/test_reward_models.py
+++ b/test/registered/models/test_reward_models.py
@@ -1,3 +1,9 @@
+from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
+
+# Reward model tests
+register_cuda_ci(est_time=103, suite="stage-b-test-small-1-gpu")
+register_amd_ci(est_time=132, suite="stage-b-test-small-1-gpu")
+
 # Copyright 2023-2024 SGLang Team
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
--- a/test/registered/models/test_transformers_models.py
+++ b/test/registered/models/test_transformers_models.py
@@ -1,3 +1,9 @@
+from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
+
+# Transformers fallback model tests
+register_cuda_ci(est_time=245, suite="stage-b-test-small-1-gpu")
+register_amd_ci(est_time=320, suite="stage-b-test-small-1-gpu")
+
 import dataclasses
 import multiprocessing as mp
 import unittest
--- a/test/registered/models/test_vlm_models.py
+++ b/test/registered/models/test_vlm_models.py
@@ -1,3 +1,9 @@
+from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
+
+# VLM (Vision Language Model) tests
+register_cuda_ci(est_time=270, suite="stage-b-test-small-1-gpu")
+register_amd_ci(est_time=387, suite="stage-b-test-small-1-gpu")
+
 import argparse
 import random
 import sys
--- a/test/run_suite.py
+++ b/test/run_suite.py
@@ -18,7 +18,7 @@ HW_MAPPING = {
 # Per-commit test suites (run on every PR)
 PER_COMMIT_SUITES = {
    HWBackend.CPU: ["default"],
-    HWBackend.AMD: ["stage-a-test-1"],
+    HWBackend.AMD: ["stage-a-test-1", "stage-b-test-small-1-gpu"],
    HWBackend.CUDA: [
        "stage-a-test-1",
        "stage-b-test-small-1-gpu",
--- a/test/srt/run_suite.py
+++ b/test/srt/run_suite.py
@@ -15,16 +15,6 @@ suites = {
        TestFile("layers/attention/mamba/test_causal_conv1d.py", 25),
        TestFile("layers/attention/mamba/test_mamba_ssm.py", 7),
        TestFile("layers/attention/mamba/test_mamba_ssm_ssd.py", 13),
-        TestFile("models/test_compressed_tensors_models.py", 42),
-        TestFile("models/test_cross_encoder_models.py", 100),
-        TestFile("models/test_embedding_models.py", 73),
-        TestFile("models/test_encoder_embedding_models.py", 221),
-        TestFile("models/test_generation_models.py", 103),
-        TestFile("models/test_nvidia_nemotron_nano_v2_vl.py", 214),  # GSM8k + MMMU
-        TestFile("models/test_qwen_models.py", 90),
-        TestFile("models/test_reward_models.py", 103),
-        TestFile("models/test_transformers_models.py", 245),
-        TestFile("models/test_vlm_models.py", 270),
        TestFile("openai_server/basic/test_openai_embedding.py", 70),
        TestFile("openai_server/basic/test_openai_server.py", 184),
        TestFile("openai_server/basic/test_protocol.py", 3),
@@ -195,12 +185,6 @@ suite_amd = {
        # TestFile("lora/test_lora_cuda_graph.py", 250), # Disabled temporarily, see https://github.com/sgl-project/sglang/issues/13107
        # TestFile("lora/test_lora_qwen3.py", 97), # Disabled temporarily, see https://github.com/sgl-project/sglang/issues/13107
        TestFile("dllm/test_llada2_mini_amd.py", 520),
-        TestFile("models/test_compressed_tensors_models.py", 42),
-        TestFile("models/test_cross_encoder_models.py", 150),
-        TestFile("models/test_qwen_models.py", 82),
-        TestFile("models/test_reward_models.py", 132),
-        TestFile("models/test_transformers_models.py", 320),
-        TestFile("models/test_vlm_models.py", 387),
        TestFile("openai_server/basic/test_openai_embedding.py", 141),
        TestFile("openai_server/basic/test_openai_server.py", 149),
        TestFile("openai_server/basic/test_protocol.py", 10),
@@ -261,7 +245,6 @@ suite_amd = {
        TestFile("test_video_utils.py", 8),
        # Disabled temporarily
        # TestFile("test_vlm_input_format.py", 300),
-        # TestFile("models/test_embedding_models.py", 73), # Disabled temporarily, see https://github.com/sgl-project/sglang/issues/11127
        # TestFile("openai_server/features/test_openai_server_hidden_states.py", 240),
        # TestFile("rl/test_update_weights_from_tensor.py", 48),
        # TestFile("test_no_overlap_scheduler.py", 234), # Disabled temporarily and track in #7703