diff --git a/.github/workflows/pr-test-amd.yml b/.github/workflows/pr-test-amd.yml index a8fde9567..a3c2133cd 100644 --- a/.github/workflows/pr-test-amd.yml +++ b/.github/workflows/pr-test-amd.yml @@ -184,6 +184,46 @@ jobs: run: | docker exec -w /sglang-checkout/test ci_sglang python3 run_suite.py --hw amd --suite stage-a-test-1 + stage-b-test-small-1-gpu-amd: + needs: [check-changes, stage-a-test-1-amd] + if: | + always() && + ( + (inputs.target_stage == 'stage-b-test-small-1-gpu-amd') || + ( + !inputs.target_stage && + (!failure() && !cancelled()) && + ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true')) + ) + ) + strategy: + fail-fast: false + matrix: + runner: [linux-mi325-gpu-1] + part: [0, 1, 2, 3] + runs-on: ${{matrix.runner}} + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }} + + - name: Ensure VRAM is clear + run: bash scripts/ensure_vram_clear.sh rocm + + - name: Start CI container + run: bash scripts/ci/amd_ci_start_container.sh + env: + GITHUB_WORKSPACE: ${{ github.workspace }} + + - name: Install dependencies + run: bash scripts/ci/amd_ci_install_dependency.sh + + - name: Run test + timeout-minutes: 30 + run: | + docker exec -w /sglang-checkout/test ci_sglang python3 run_suite.py --hw amd --suite stage-b-test-small-1-gpu --auto-partition-id ${{ matrix.part }} --auto-partition-size 4 + multimodal-gen-test-1-gpu-amd: needs: [check-changes] if: needs.check-changes.outputs.multimodal_gen == 'true' @@ -754,7 +794,6 @@ jobs: run: | bash scripts/ci/amd_ci_exec.sh -e SGLANG_USE_AITER=0 python3 test_eval_accuracy_large.py bash scripts/ci/amd_ci_exec.sh python3 test_eval_fp8_accuracy.py - bash scripts/ci/amd_ci_exec.sh python3 models/test_qwen_models.py accuracy-test-2-gpu-amd: needs: [check-changes, accuracy-test-1-gpu-amd] @@ -844,6 +883,7 @@ jobs: multimodal-gen-test-2-gpu-amd, stage-a-test-1-amd, + stage-b-test-small-1-gpu-amd, unit-test-backend-1-gpu-amd, unit-test-backend-2-gpu-amd, unit-test-backend-8-gpu-amd, diff --git a/.github/workflows/pr-test.yml b/.github/workflows/pr-test.yml index 242d64591..f6b70e5c3 100644 --- a/.github/workflows/pr-test.yml +++ b/.github/workflows/pr-test.yml @@ -532,7 +532,7 @@ jobs: strategy: fail-fast: false matrix: - partition: [0, 1, 2, 3, 4] + partition: [0, 1, 2, 3, 4, 5, 6, 7] steps: - name: Checkout code uses: actions/checkout@v4 @@ -559,7 +559,7 @@ jobs: if [[ "${{ needs.check-changes.outputs.continue_on_error }}" == "true" ]]; then CONTINUE_ON_ERROR_FLAG="--continue-on-error" fi - python3 run_suite.py --hw cuda --suite stage-b-test-small-1-gpu --auto-partition-id ${{ matrix.partition }} --auto-partition-size 5 $CONTINUE_ON_ERROR_FLAG + python3 run_suite.py --hw cuda --suite stage-b-test-small-1-gpu --auto-partition-id ${{ matrix.partition }} --auto-partition-size 8 $CONTINUE_ON_ERROR_FLAG stage-b-test-large-1-gpu: needs: [check-changes, call-gate, stage-a-test-1, sgl-kernel-build-wheels] @@ -891,7 +891,7 @@ jobs: fail-fast: false max-parallel: ${{ fromJson(needs.check-changes.outputs.max_parallel) }} matrix: - part: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12] + part: [0, 1, 2, 3, 4, 5] steps: - name: Checkout code uses: actions/checkout@v4 @@ -922,7 +922,7 @@ jobs: if [[ "${{ needs.check-changes.outputs.continue_on_error }}" == "true" ]]; then CONTINUE_ON_ERROR_FLAG="--continue-on-error" fi - python3 run_suite.py --suite per-commit-1-gpu --auto-partition-id ${{ matrix.part }} --auto-partition-size 13 $RETRY_FLAG $CONTINUE_ON_ERROR_FLAG + python3 run_suite.py --suite per-commit-1-gpu --auto-partition-id ${{ matrix.part }} --auto-partition-size 6 $RETRY_FLAG $CONTINUE_ON_ERROR_FLAG stage-b-test-4-gpu-b200: diff --git a/docker/xeon.Dockerfile b/docker/xeon.Dockerfile index da2f4a0f0..52ca31e43 100644 --- a/docker/xeon.Dockerfile +++ b/docker/xeon.Dockerfile @@ -45,6 +45,7 @@ RUN source $HOME/.local/bin/env && \ cp pyproject_cpu.toml pyproject.toml && \ uv pip install . && \ uv pip install torch==${VER_TORCH} torchvision==${VER_TORCHVISION} triton==${VER_TRITON} --force-reinstall && \ + uv pip install tabulate && \ cd ../sgl-kernel && \ cp pyproject_cpu.toml pyproject.toml && \ uv pip install . diff --git a/docker/xpu.Dockerfile b/docker/xpu.Dockerfile index 5aa57b3d1..0fa726632 100644 --- a/docker/xpu.Dockerfile +++ b/docker/xpu.Dockerfile @@ -60,7 +60,7 @@ RUN --mount=type=secret,id=github_token \ cp pyproject_xpu.toml pyproject.toml && \ pip install . && \ pip install xgrammar --no-deps && \ - pip install msgspec blake3 py-cpuinfo compressed_tensors gguf partial_json_parser einops --root-user-action=ignore && \ + pip install msgspec blake3 py-cpuinfo compressed_tensors gguf partial_json_parser einops tabulate --root-user-action=ignore && \ conda install libsqlite=3.48.0 -y && \ # Add environment setup commands to .bashrc again (in case it was overwritten) echo ". /home/sdp/miniforge3/bin/activate; conda activate py${PYTHON_VERSION}; cd /home/sdp" >> /home/sdp/.bashrc diff --git a/scripts/ci/slash_command_handler.py b/scripts/ci/slash_command_handler.py index f251e4a36..537956f8d 100644 --- a/scripts/ci/slash_command_handler.py +++ b/scripts/ci/slash_command_handler.py @@ -176,6 +176,7 @@ def handle_rerun_stage( amd_stages = [ "sgl-kernel-unit-test-amd", "stage-a-test-1-amd", + "stage-b-test-small-1-gpu-amd", "unit-test-backend-1-gpu-amd", "unit-test-backend-2-gpu-amd", "unit-test-backend-8-gpu-amd", diff --git a/test/srt/models/test_compressed_tensors_models.py b/test/registered/models/test_compressed_tensors_models.py similarity index 83% rename from test/srt/models/test_compressed_tensors_models.py rename to test/registered/models/test_compressed_tensors_models.py index 34f699de4..e86b74b65 100644 --- a/test/srt/models/test_compressed_tensors_models.py +++ b/test/registered/models/test_compressed_tensors_models.py @@ -1,3 +1,9 @@ +from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci + +# Model tests for compressed tensors (FP8) +register_cuda_ci(est_time=42, suite="stage-b-test-small-1-gpu") +register_amd_ci(est_time=42, suite="stage-b-test-small-1-gpu") + import unittest from types import SimpleNamespace diff --git a/test/srt/models/test_cross_encoder_models.py b/test/registered/models/test_cross_encoder_models.py similarity index 91% rename from test/srt/models/test_cross_encoder_models.py rename to test/registered/models/test_cross_encoder_models.py index 6ff963dd4..7fc5f4659 100644 --- a/test/srt/models/test_cross_encoder_models.py +++ b/test/registered/models/test_cross_encoder_models.py @@ -1,3 +1,9 @@ +from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci + +# Cross encoder model tests +register_cuda_ci(est_time=100, suite="stage-b-test-small-1-gpu") +register_amd_ci(est_time=150, suite="stage-b-test-small-1-gpu") + import multiprocessing as mp import random import unittest diff --git a/test/srt/models/test_embedding_models.py b/test/registered/models/test_embedding_models.py similarity index 94% rename from test/srt/models/test_embedding_models.py rename to test/registered/models/test_embedding_models.py index d046166ae..440ae57a3 100644 --- a/test/srt/models/test_embedding_models.py +++ b/test/registered/models/test_embedding_models.py @@ -1,3 +1,13 @@ +from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci + +# Embedding model tests +register_cuda_ci(est_time=73, suite="stage-b-test-small-1-gpu") +register_amd_ci( + est_time=73, + suite="stage-b-test-small-1-gpu", + disabled="see https://github.com/sgl-project/sglang/issues/11127", +) + # Copyright 2023-2024 SGLang Team # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/test/srt/models/test_encoder_embedding_models.py b/test/registered/models/test_encoder_embedding_models.py similarity index 97% rename from test/srt/models/test_encoder_embedding_models.py rename to test/registered/models/test_encoder_embedding_models.py index dafaa72db..8aa60013d 100644 --- a/test/srt/models/test_encoder_embedding_models.py +++ b/test/registered/models/test_encoder_embedding_models.py @@ -1,3 +1,8 @@ +from sglang.test.ci.ci_register import register_cuda_ci + +# Encoder embedding model tests (CUDA only) +register_cuda_ci(est_time=221, suite="stage-b-test-small-1-gpu") + # Copyright 2023-2024 SGLang Team # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/test/srt/models/test_generation_models.py b/test/registered/models/test_generation_models.py similarity index 97% rename from test/srt/models/test_generation_models.py rename to test/registered/models/test_generation_models.py index 43770d5a1..bc45d1918 100644 --- a/test/srt/models/test_generation_models.py +++ b/test/registered/models/test_generation_models.py @@ -1,3 +1,8 @@ +from sglang.test.ci.ci_register import register_cuda_ci + +# Generation model tests (CUDA only) +register_cuda_ci(est_time=103, suite="stage-b-test-small-1-gpu") + # Copyright 2023-2024 SGLang Team # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/test/srt/models/test_nvidia_nemotron_nano_v2_vl.py b/test/registered/models/test_nvidia_nemotron_nano_v2_vl.py similarity index 82% rename from test/srt/models/test_nvidia_nemotron_nano_v2_vl.py rename to test/registered/models/test_nvidia_nemotron_nano_v2_vl.py index e41aedd67..cc20ff42a 100644 --- a/test/srt/models/test_nvidia_nemotron_nano_v2_vl.py +++ b/test/registered/models/test_nvidia_nemotron_nano_v2_vl.py @@ -1,3 +1,9 @@ +from sglang.test.ci.ci_register import register_cuda_ci + +# NVIDIA Nemotron Nano V2 VL model tests (CUDA only) +# GSM8k + MMMU evaluation +register_cuda_ci(est_time=214, suite="stage-b-test-small-1-gpu") + import unittest from sglang.test.kits.gsm8k_accuracy_kit import GSM8KMixin diff --git a/test/srt/models/test_qwen_models.py b/test/registered/models/test_qwen_models.py similarity index 90% rename from test/srt/models/test_qwen_models.py rename to test/registered/models/test_qwen_models.py index 567e19f08..026bb9ef2 100644 --- a/test/srt/models/test_qwen_models.py +++ b/test/registered/models/test_qwen_models.py @@ -1,3 +1,9 @@ +from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci + +# Qwen model tests +register_cuda_ci(est_time=90, suite="stage-b-test-small-1-gpu") +register_amd_ci(est_time=82, suite="stage-b-test-small-1-gpu") + import unittest from types import SimpleNamespace diff --git a/test/srt/models/test_reward_models.py b/test/registered/models/test_reward_models.py similarity index 94% rename from test/srt/models/test_reward_models.py rename to test/registered/models/test_reward_models.py index 72b74eb33..3a197c421 100644 --- a/test/srt/models/test_reward_models.py +++ b/test/registered/models/test_reward_models.py @@ -1,3 +1,9 @@ +from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci + +# Reward model tests +register_cuda_ci(est_time=103, suite="stage-b-test-small-1-gpu") +register_amd_ci(est_time=132, suite="stage-b-test-small-1-gpu") + # Copyright 2023-2024 SGLang Team # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/test/srt/models/test_transformers_models.py b/test/registered/models/test_transformers_models.py similarity index 95% rename from test/srt/models/test_transformers_models.py rename to test/registered/models/test_transformers_models.py index 746f78087..782b7a6c6 100644 --- a/test/srt/models/test_transformers_models.py +++ b/test/registered/models/test_transformers_models.py @@ -1,3 +1,9 @@ +from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci + +# Transformers fallback model tests +register_cuda_ci(est_time=245, suite="stage-b-test-small-1-gpu") +register_amd_ci(est_time=320, suite="stage-b-test-small-1-gpu") + import dataclasses import multiprocessing as mp import unittest diff --git a/test/srt/models/test_vlm_models.py b/test/registered/models/test_vlm_models.py similarity index 86% rename from test/srt/models/test_vlm_models.py rename to test/registered/models/test_vlm_models.py index c195bbd81..58b6a7a01 100644 --- a/test/srt/models/test_vlm_models.py +++ b/test/registered/models/test_vlm_models.py @@ -1,3 +1,9 @@ +from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci + +# VLM (Vision Language Model) tests +register_cuda_ci(est_time=270, suite="stage-b-test-small-1-gpu") +register_amd_ci(est_time=387, suite="stage-b-test-small-1-gpu") + import argparse import random import sys diff --git a/test/run_suite.py b/test/run_suite.py index 8cf37b20b..57ae2c505 100644 --- a/test/run_suite.py +++ b/test/run_suite.py @@ -18,7 +18,7 @@ HW_MAPPING = { # Per-commit test suites (run on every PR) PER_COMMIT_SUITES = { HWBackend.CPU: ["default"], - HWBackend.AMD: ["stage-a-test-1"], + HWBackend.AMD: ["stage-a-test-1", "stage-b-test-small-1-gpu"], HWBackend.CUDA: [ "stage-a-test-1", "stage-b-test-small-1-gpu", diff --git a/test/srt/run_suite.py b/test/srt/run_suite.py index eea5b3468..70fb9cdf6 100644 --- a/test/srt/run_suite.py +++ b/test/srt/run_suite.py @@ -15,16 +15,6 @@ suites = { TestFile("layers/attention/mamba/test_causal_conv1d.py", 25), TestFile("layers/attention/mamba/test_mamba_ssm.py", 7), TestFile("layers/attention/mamba/test_mamba_ssm_ssd.py", 13), - TestFile("models/test_compressed_tensors_models.py", 42), - TestFile("models/test_cross_encoder_models.py", 100), - TestFile("models/test_embedding_models.py", 73), - TestFile("models/test_encoder_embedding_models.py", 221), - TestFile("models/test_generation_models.py", 103), - TestFile("models/test_nvidia_nemotron_nano_v2_vl.py", 214), # GSM8k + MMMU - TestFile("models/test_qwen_models.py", 90), - TestFile("models/test_reward_models.py", 103), - TestFile("models/test_transformers_models.py", 245), - TestFile("models/test_vlm_models.py", 270), TestFile("openai_server/basic/test_openai_embedding.py", 70), TestFile("openai_server/basic/test_openai_server.py", 184), TestFile("openai_server/basic/test_protocol.py", 3), @@ -195,12 +185,6 @@ suite_amd = { # TestFile("lora/test_lora_cuda_graph.py", 250), # Disabled temporarily, see https://github.com/sgl-project/sglang/issues/13107 # TestFile("lora/test_lora_qwen3.py", 97), # Disabled temporarily, see https://github.com/sgl-project/sglang/issues/13107 TestFile("dllm/test_llada2_mini_amd.py", 520), - TestFile("models/test_compressed_tensors_models.py", 42), - TestFile("models/test_cross_encoder_models.py", 150), - TestFile("models/test_qwen_models.py", 82), - TestFile("models/test_reward_models.py", 132), - TestFile("models/test_transformers_models.py", 320), - TestFile("models/test_vlm_models.py", 387), TestFile("openai_server/basic/test_openai_embedding.py", 141), TestFile("openai_server/basic/test_openai_server.py", 149), TestFile("openai_server/basic/test_protocol.py", 10), @@ -261,7 +245,6 @@ suite_amd = { TestFile("test_video_utils.py", 8), # Disabled temporarily # TestFile("test_vlm_input_format.py", 300), - # TestFile("models/test_embedding_models.py", 73), # Disabled temporarily, see https://github.com/sgl-project/sglang/issues/11127 # TestFile("openai_server/features/test_openai_server_hidden_states.py", 240), # TestFile("rl/test_update_weights_from_tensor.py", 48), # TestFile("test_no_overlap_scheduler.py", 234), # Disabled temporarily and track in #7703