[AMD CI] migrate and re-enable CI tests to new CI registry (#16949)

Co-authored-by: yctseng0211 <yctseng@amd.com>

.github/workflows/pr-test-amd.yml

@@ -634,10 +634,18 @@ jobs:
         run: |
           bash scripts/ci/amd_ci_exec.sh python3 run_suite.py --suite per-commit-8-gpu-amd --auto-partition-id ${{ matrix.part }} --auto-partition-size 2 --timeout-per-file 3600
 
-  unit-test-backend-8-gpu-amd-mi35x:
-    needs: [check-changes, stage-a-test-1-amd]
-    if: always() && !failure() && !cancelled() &&
-      ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
+  stage-c-test-large-8-gpu-amd-mi35x:
+    needs: [check-changes, call-gate, stage-b-test-small-1-gpu-amd, stage-b-test-large-2-gpu-amd]
+    if: |
+      always() &&
+      (
+        (inputs.target_stage == 'stage-c-test-large-8-gpu-amd-mi35x') ||
+        (
+          !inputs.target_stage &&
+          (!failure() && !cancelled()) &&
+          ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
+        )
+      )
     strategy:
       fail-fast: false
       matrix:
@@ -646,6 +654,8 @@ jobs:
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }}
 
       - name: Ensure VRAM is clear
         run: bash scripts/ensure_vram_clear.sh rocm
@@ -661,7 +671,7 @@ jobs:
       - name: Run test
         timeout-minutes: 60
         run: |
-          bash scripts/ci/amd_ci_exec.sh python3 run_suite.py --suite per-commit-8-gpu-amd-mi35x --timeout-per-file 3600
+          bash scripts/ci/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-c-test-large-8-gpu-amd-mi35x --timeout-per-file 3600
 
   performance-test-1-gpu-part-1-amd:
     needs: [check-changes, stage-a-test-1-amd]
@@ -931,7 +941,7 @@ jobs:
         stage-b-test-large-2-gpu-amd,
         unit-test-backend-1-gpu-amd,
         unit-test-backend-8-gpu-amd,
-        unit-test-backend-8-gpu-amd-mi35x,
+        stage-c-test-large-8-gpu-amd-mi35x,
         performance-test-1-gpu-part-1-amd,
         performance-test-1-gpu-part-2-amd,
         performance-test-2-gpu-amd,

scripts/ci/slash_command_handler.py

@@ -258,6 +258,7 @@ def handle_rerun_stage(
         "stage-b-test-small-1-gpu-amd",
         "stage-b-test-small-1-gpu-amd-mi35x",
         "stage-b-test-large-2-gpu-amd",
+        "stage-c-test-large-8-gpu-amd-mi35x",
         "unit-test-backend-1-gpu-amd",
         "unit-test-backend-2-gpu-amd",
         "unit-test-backend-8-gpu-amd",

test/registered/amd/test_deepseek_r1_mxfp4_8gpu.py

@@ -5,6 +5,7 @@ from types import SimpleNamespace
 import requests
 
 from sglang.srt.utils import kill_process_tree
+from sglang.test.ci.ci_register import register_amd_ci
 from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k
 from sglang.test.send_one import BenchArgs, send_one_prompt
 from sglang.test.test_utils import (
@@ -15,6 +16,8 @@ from sglang.test.test_utils import (
     write_github_step_summary,
 )
 
+register_amd_ci(est_time=3600, suite="stage-c-test-large-8-gpu-amd-mi35x")
+
 DEEPSEEK_R1_MODEL_PATH = "amd/DeepSeek-R1-MXFP4-Preview"
 SERVER_LAUNCH_TIMEOUT = 1200
 

test/registered/attention/test_mamba_unittest.py

@@ -12,9 +12,10 @@ from sglang.srt.mem_cache.memory_pool import HybridLinearKVPool, HybridReqToToke
 from sglang.srt.mem_cache.radix_cache import RadixKey
 from sglang.srt.sampling.sampling_params import SamplingParams
 from sglang.srt.server_args import ServerArgs, set_global_server_args_for_scheduler
-from sglang.test.ci.ci_register import register_cuda_ci
+from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
 
 register_cuda_ci(est_time=9, suite="stage-b-test-small-1-gpu")
+register_amd_ci(est_time=9, suite="stage-b-test-small-1-gpu-amd")
 register_cuda_ci(est_time=7, suite="stage-b-test-small-1-gpu-5090")
 
 

test/registered/attention/test_radix_cache_unit.py

@@ -17,10 +17,11 @@ Usage:
     python -m pytest test_radix_cache_unit.py::TestRadixCache::test_insert_basic
 """
 
-from sglang.test.ci.ci_register import register_cuda_ci
+from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
 
 # CPU-based unit test, runs quickly on any GPU runner
 register_cuda_ci(est_time=5, suite="stage-b-test-small-1-gpu")
+register_amd_ci(est_time=5, suite="stage-b-test-small-1-gpu-amd")
 register_cuda_ci(est_time=7, suite="stage-b-test-small-1-gpu-5090")
 
 import time

test/registered/core/test_hidden_states.py

@@ -4,12 +4,20 @@ import torch
 from transformers import AutoModelForCausalLM, AutoTokenizer
 
 import sglang as sgl
-from sglang.test.ci.ci_register import register_cuda_ci
+from sglang.srt.utils import is_hip
+from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
 from sglang.test.test_utils import DEFAULT_SMALL_MODEL_NAME_FOR_TEST, CustomTestCase
 
 register_cuda_ci(est_time=55, suite="stage-b-test-small-1-gpu")
+register_amd_ci(est_time=55, suite="stage-b-test-small-1-gpu-amd")
 register_cuda_ci(est_time=31, suite="stage-b-test-small-1-gpu-5090")
 
+_is_hip = is_hip()
+if _is_hip:
+    import os
+
+    os.environ["SGLANG_USE_AITER"] = "0"
+
 
 class TestHiddenState(CustomTestCase):
     def test_return_hidden_states(self):

test/registered/dllm/test_llada2_mini.py

@@ -1,6 +1,7 @@
-from sglang.test.ci.ci_register import register_cuda_ci
+from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
 
 register_cuda_ci(est_time=181, suite="stage-b-test-small-1-gpu")
+register_amd_ci(est_time=330, suite="stage-b-test-small-1-gpu-amd")
 
 import unittest
 from types import SimpleNamespace
@@ -62,7 +63,10 @@ class TestLLaDA2Mini(CustomTestCase):
         print(f"{metrics=}")
 
         self.assertGreater(metrics["accuracy"], 0.88)
-        self.assertGreater(metrics["output_throughput"], 150)
+        if is_in_amd_ci():
+            self.assertGreater(metrics["output_throughput"], 80)
+        else:
+            self.assertGreater(metrics["output_throughput"], 150)
 
     def test_bs_1_speed(self):
         args = BenchArgs(port=int(self.base_url.split(":")[-1]), max_new_tokens=2048)

test/registered/hicache/test_hicache_storage.py

@@ -1,6 +1,7 @@
-from sglang.test.ci.ci_register import register_cuda_ci
+from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
 
 register_cuda_ci(est_time=96, suite="stage-b-test-small-1-gpu")
+register_amd_ci(est_time=300, suite="stage-b-test-small-1-gpu-amd")
 register_cuda_ci(est_time=96, suite="stage-b-test-small-1-gpu-5090")
 
 import time

test/registered/hicache/test_hicache_variants.py

@@ -1,7 +1,7 @@
-from sglang.test.ci.ci_register import register_cuda_ci
+from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
 
 register_cuda_ci(est_time=524, suite="stage-b-test-small-1-gpu")
-
+register_amd_ci(est_time=524, suite="stage-b-test-small-1-gpu-amd")
 """
 Consolidated HiCache variant tests.
 Tests HiCache with different configurations: standard, MLA, EAGLE, and page size variants.
@@ -114,6 +114,7 @@ class TestHiCacheMLA(HiCacheBaseServer, HiCacheEvalMixin, HiCacheMGSMEvalMixin):
     expected_mmlu_score = 0.5
 
 
+@unittest.skipIf(is_hip(), "Disabled for AMD-aiter")
 class TestHiCacheEagle(HiCacheBaseServer, HiCacheEvalMixin):
     """HiCache with EAGLE speculative decoding tests"""
 

test/registered/layers/mamba/test_causal_conv1d.py

@@ -1,6 +1,7 @@
-from sglang.test.ci.ci_register import register_cuda_ci
+from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
 
 register_cuda_ci(est_time=25, suite="stage-b-test-small-1-gpu")
+register_amd_ci(est_time=25, suite="stage-b-test-small-1-gpu-amd")
 register_cuda_ci(est_time=3, suite="stage-b-test-small-1-gpu-5090")
 
 # Adapted from https://github.com/vllm-project/vllm/blob/main/tests/kernels/mamba/test_causal_conv1d.py

test/registered/layers/mamba/test_mamba_ssm.py

@@ -1,6 +1,7 @@
-from sglang.test.ci.ci_register import register_cuda_ci
+from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
 
 register_cuda_ci(est_time=7, suite="stage-b-test-small-1-gpu")
+register_amd_ci(est_time=20, suite="stage-b-test-small-1-gpu-amd")
 register_cuda_ci(est_time=5, suite="stage-b-test-small-1-gpu-5090")
 
 # Adapted from https://github.com/vllm-project/vllm/blob/633f943e30a4444d890d26b81850f7217736f840/tests/kernels/mamba/test_mamba_ssm_ssd.py

test/registered/layers/mamba/test_mamba_ssm_ssd.py

@@ -1,6 +1,7 @@
-from sglang.test.ci.ci_register import register_cuda_ci
+from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
 
 register_cuda_ci(est_time=13, suite="stage-b-test-small-1-gpu")
+register_amd_ci(est_time=30, suite="stage-b-test-small-1-gpu-amd")
 register_cuda_ci(est_time=8, suite="stage-b-test-small-1-gpu-5090")
 
 # Adapted from https://github.com/vllm-project/vllm/blob/633f943e30a4444d890d26b81850f7217736f840/tests/kernels/mamba/test_mamba_ssm_ssd.py

test/registered/lora/test_lora_hf_sgl_logprob_diff.py

@@ -34,13 +34,17 @@ from typing import Any, Dict, List, Optional, Tuple
 import numpy as np
 import torch
 
-from sglang.test.ci.ci_register import register_cuda_ci
+from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
 from sglang.test.runners import HFRunner, SRTRunner
 
 register_cuda_ci(
     est_time=150,
     suite="stage-b-test-small-1-gpu",
 )
+register_amd_ci(
+    est_time=250,
+    suite="stage-b-test-small-1-gpu-amd",
+)
 register_cuda_ci(
     est_time=76,
     suite="stage-b-test-small-1-gpu-5090",

test/registered/mla/test_mla.py

@@ -14,11 +14,7 @@ from sglang.test.test_utils import (
 
 # MLA attention test with MGSM evaluation
 register_cuda_ci(est_time=194, suite="stage-b-test-small-1-gpu")
-register_amd_ci(
-    est_time=242,
-    suite="stage-b-test-small-1-gpu-amd",
-    disabled="see https://github.com/sgl-project/sglang/issues/13107",
-)
+register_amd_ci(est_time=1100, suite="stage-b-test-small-1-gpu-amd")
 
 
 class TestMLA(CustomTestCase):

test/registered/mla/test_mla_fp8.py

@@ -2,7 +2,7 @@ import unittest
 from types import SimpleNamespace
 
 from sglang.srt.utils import kill_process_tree
-from sglang.test.ci.ci_register import register_cuda_ci
+from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
 from sglang.test.run_eval import run_eval
 from sglang.test.test_utils import (
     DEFAULT_MLA_FP8_MODEL_NAME_FOR_TEST,
@@ -14,6 +14,7 @@ from sglang.test.test_utils import (
 
 # MLA FP8 KV cache test with MGSM evaluation
 register_cuda_ci(est_time=77, suite="stage-b-test-small-1-gpu")
+register_amd_ci(est_time=360, suite="stage-b-test-small-1-gpu-amd")
 
 
 class TestMLA(CustomTestCase):

test/registered/model_loading/test_external_models.py

@@ -7,7 +7,7 @@ from sglang.test.test_utils import CustomTestCase
 
 register_cuda_ci(est_time=30, suite="stage-b-test-small-1-gpu")
 register_cuda_ci(est_time=26, suite="stage-b-test-small-1-gpu-5090")
-register_amd_ci(est_time=45, suite="stage-b-test-small-1-gpu")
+register_amd_ci(est_time=45, suite="stage-b-test-small-1-gpu-amd")
 
 
 class TestExternalModels(CustomTestCase):

test/registered/model_loading/test_modelopt_export.py

@@ -17,9 +17,10 @@ from sglang.srt.configs.device_config import DeviceConfig
 from sglang.srt.configs.load_config import LoadConfig
 from sglang.srt.configs.model_config import ModelConfig
 from sglang.srt.model_loader.loader import ModelOptModelLoader
-from sglang.test.ci.ci_register import register_cuda_ci
+from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
 
 register_cuda_ci(est_time=9, suite="stage-b-test-small-1-gpu")
+register_amd_ci(est_time=9, suite="stage-b-test-small-1-gpu-amd")
 
 # Note: PYTHONPATH=python should be set when running tests
 

test/registered/models/test_generation_models.py

@@ -1,7 +1,8 @@
-from sglang.test.ci.ci_register import register_cuda_ci
+from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
 
 # Generation model tests (CUDA only)
 register_cuda_ci(est_time=103, suite="stage-b-test-small-1-gpu")
+register_amd_ci(est_time=106, suite="stage-b-test-small-1-gpu-amd")
 
 # Copyright 2023-2024 SGLang Team
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -28,10 +29,11 @@ import dataclasses
 import multiprocessing as mp
 import os
 import unittest
-from typing import List
+from typing import List, Optional
 
 import torch
 
+from sglang.srt.utils import is_hip
 from sglang.test.runners import (
     DEFAULT_PROMPTS,
     HFRunner,
@@ -50,12 +52,13 @@ class ModelCase:
     rouge_l_tolerance: float = 1
     skip_long_prompt: bool = False
     trust_remote_code: bool = False
+    attention_backend: Optional[str] = None
 
 
 # Popular models that run on the CI
 CI_MODELS = [
     ModelCase("meta-llama/Llama-3.1-8B-Instruct"),
-    ModelCase("google/gemma-2-2b"),
+    ModelCase("google/gemma-2-2b", attention_backend="triton" if is_hip() else None),
 ]
 
 # the complete set of models to test sglang's generation model
@@ -145,6 +148,7 @@ class TestGenerationModels(CustomTestCase):
             torch_dtype=torch_dtype,
             model_type="generation",
             trust_remote_code=model_case.trust_remote_code,
+            attention_backend=model_case.attention_backend,
         ) as srt_runner:
             srt_outputs = srt_runner.forward(prompts, max_new_tokens=max_new_tokens)
 

test/registered/quant/test_torchao.py

@@ -4,9 +4,10 @@ from types import SimpleNamespace
 import requests
 
 from sglang import Engine
-from sglang.test.ci.ci_register import register_cuda_ci
+from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
 
 register_cuda_ci(est_time=103, suite="stage-b-test-small-1-gpu")
+register_amd_ci(est_time=106, suite="stage-b-test-small-1-gpu-amd")
 register_cuda_ci(est_time=88, suite="stage-b-test-small-1-gpu-5090")
 from sglang.lang.chat_template import get_chat_template_by_model_path
 from sglang.srt.utils import kill_process_tree
@@ -18,6 +19,7 @@ from sglang.test.test_utils import (
     DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
     DEFAULT_URL_FOR_TEST,
     CustomTestCase,
+    is_in_amd_ci,
     popen_launch_server,
 )
 
@@ -75,7 +77,10 @@ class TestTorchAO(CustomTestCase):
         print(res["text"])
         throughput = max_tokens / (tok - tic)
         print(f"Throughput: {throughput} tokens/s")
-        assert throughput >= 210
+        if is_in_amd_ci():
+            assert throughput >= 150
+        else:
+            assert throughput >= 210
 
 
 class TestTorchAOForVLM(CustomTestCase):

test/registered/rl/test_update_weights_from_tensor.py

@@ -1,6 +1,7 @@
-from sglang.test.ci.ci_register import register_cuda_ci
+from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
 
 register_cuda_ci(est_time=195, suite="stage-b-test-small-1-gpu")
+register_amd_ci(est_time=195, suite="stage-b-test-small-1-gpu-amd")
 register_cuda_ci(est_time=164, suite="stage-b-test-small-1-gpu-5090")
 
 import gc

test/registered/scheduler/test_no_overlap_scheduler.py

@@ -6,10 +6,11 @@ python3 test_overlap_schedule.py
 
 import unittest
 
-from sglang.test.ci.ci_register import register_cuda_ci
+from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
 from sglang.test.test_utils import CustomTestCase, run_mmlu_test
 
 register_cuda_ci(est_time=245, suite="stage-b-test-small-1-gpu")
+register_amd_ci(est_time=275, suite="stage-b-test-small-1-gpu-amd")
 
 
 class TestOverlapSchedule(CustomTestCase):

test/registered/spec/utils/test_build_eagle_tree.py

@@ -6,9 +6,10 @@ from sglang.srt.speculative.eagle_utils import (
     build_tree_kernel_efficient,
     organize_draft_results,
 )
-from sglang.test.ci.ci_register import register_cuda_ci
+from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
 
 register_cuda_ci(est_time=3, suite="stage-b-test-small-1-gpu")
+register_amd_ci(est_time=3, suite="stage-b-test-small-1-gpu-amd")
 register_cuda_ci(est_time=2, suite="stage-b-test-small-1-gpu-5090")
 
 

test/registered/vlm/test_evs.py

@@ -3,10 +3,11 @@ from types import SimpleNamespace
 
 import pytest
 
-from sglang.test.ci.ci_register import register_cuda_ci
+from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
 from sglang.test.test_utils import run_doctests
 
 register_cuda_ci(est_time=20, suite="stage-b-test-small-1-gpu")
+register_amd_ci(est_time=20, suite="stage-b-test-small-1-gpu-amd")
 register_cuda_ci(est_time=7, suite="stage-b-test-small-1-gpu-5090")
 
 

test/registered/vlm/test_vision_chunked_prefill.py

@@ -1,7 +1,7 @@
-from sglang.test.ci.ci_register import register_cuda_ci
+from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
 
 register_cuda_ci(est_time=150, suite="stage-b-test-small-1-gpu")
-
+register_amd_ci(est_time=270, suite="stage-b-test-small-1-gpu-amd")
 """
 Usage:
 python3 -m unittest test_vision_chunked_prefill.TestVisionChunkedPrefill.test_chunked_prefill

test/run_suite.py

@@ -23,6 +23,7 @@ PER_COMMIT_SUITES = {
         "stage-b-test-small-1-gpu-amd",
         "stage-b-test-small-1-gpu-amd-mi35x",
         "stage-b-test-large-2-gpu-amd",
+        "stage-c-test-large-8-gpu-amd-mi35x",
     ],
     HWBackend.CUDA: [
         "stage-a-test-1",

test/srt/run_suite.py

@@ -109,9 +109,6 @@ suite_amd = {
         TestFile("test_deepseek_v3_basic.py", 275),
         TestFile("test_deepseek_v3_mtp.py", 275),
     ],
-    "per-commit-8-gpu-amd-mi35x": [
-        TestFile("test_deepseek_r1_mxfp4_8gpu.py", 3600),
-    ],
     # NOTE: AMD nightly suites (nightly-amd, nightly-amd-vlm, nightly-amd-8-gpu)
     # have been migrated to test/registered/amd/nightly/ and are now managed
     # by test/run_suite.py using the registry system.