[CI] Migrate LoRA tests to test/registered/lora/ (#15176)

test/registered/lora/test_lora.py

@@ -0,0 +1,53 @@
+# Copyright 2023-2024 SGLang Team
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+import multiprocessing as mp
+import os
+import unittest
+
+from sglang.test.ci.ci_register import register_cuda_ci
+from sglang.test.lora_utils import (
+    ALL_OTHER_MULTI_LORA_MODELS,
+    CI_MULTI_LORA_MODELS,
+    run_lora_multiple_batch_on_model_cases,
+)
+from sglang.test.test_utils import CustomTestCase, is_in_ci
+
+register_cuda_ci(est_time=82, suite="stage-b-test-small-1-gpu")
+
+
+class TestLoRA(CustomTestCase):
+    def test_ci_lora_models(self):
+        run_lora_multiple_batch_on_model_cases(CI_MULTI_LORA_MODELS)
+
+    def test_all_lora_models(self):
+        if is_in_ci():
+            return
+
+        filtered_models = []
+        for model_case in ALL_OTHER_MULTI_LORA_MODELS:
+            if "ONLY_RUN" in os.environ and os.environ["ONLY_RUN"] != model_case.base:
+                continue
+            filtered_models.append(model_case)
+
+        run_lora_multiple_batch_on_model_cases(filtered_models)
+
+
+if __name__ == "__main__":
+    try:
+        mp.set_start_method("spawn")
+    except RuntimeError:
+        pass
+
+    unittest.main(warnings="ignore")

test/registered/lora/test_lora_backend.py

@@ -0,0 +1,78 @@
+# Copyright 2023-2024 SGLang Team
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+import multiprocessing as mp
+import os
+import unittest
+from typing import List
+
+from sglang.test.ci.ci_register import register_cuda_ci
+from sglang.test.lora_utils import (
+    ALL_OTHER_LORA_MODELS,
+    BACKENDS,
+    CI_LORA_MODELS,
+    DEFAULT_PROMPTS,
+    TORCH_DTYPES,
+    LoRAModelCase,
+    run_lora_test_one_by_one,
+)
+from sglang.test.test_utils import CustomTestCase, is_in_ci
+
+register_cuda_ci(est_time=200, suite="stage-b-test-small-1-gpu")
+
+
+class TestLoRABackend(CustomTestCase):
+
+    def _run_backend_on_model_cases(self, model_cases: List[LoRAModelCase]):
+        for model_case in model_cases:
+            # If skip_long_prompt is True, filter out prompts longer than 1000 characters
+            prompts = (
+                DEFAULT_PROMPTS
+                if not model_case.skip_long_prompt
+                else [p for p in DEFAULT_PROMPTS if len(p) < 1000]
+            )
+            for torch_dtype in TORCH_DTYPES:
+                for backend in BACKENDS:
+                    run_lora_test_one_by_one(
+                        prompts,
+                        model_case,
+                        torch_dtype,
+                        max_new_tokens=32,
+                        backend=backend,
+                    )
+
+    def test_ci_lora_models(self):
+        self._run_backend_on_model_cases(CI_LORA_MODELS)
+
+    def test_all_lora_models(self):
+        if is_in_ci():
+            return
+
+        # Retain ONLY_RUN check here
+        filtered_models = []
+        for model_case in ALL_OTHER_LORA_MODELS:
+            if "ONLY_RUN" in os.environ and os.environ["ONLY_RUN"] != model_case.base:
+                continue
+            filtered_models.append(model_case)
+
+        self._run_backend_on_model_cases(filtered_models)
+
+
+if __name__ == "__main__":
+    try:
+        mp.set_start_method("spawn")
+    except RuntimeError:
+        pass
+
+    unittest.main(warnings="ignore")

test/registered/lora/test_lora_eviction.py

@@ -0,0 +1,157 @@
+# Copyright 2023-2024 SGLang Team
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+import contextlib
+import multiprocessing as mp
+import unittest
+from typing import Dict, List, Tuple
+
+import torch
+
+from sglang.test.ci.ci_register import register_cuda_ci
+from sglang.test.runners import SRTRunner
+from sglang.test.test_utils import CustomTestCase
+
+register_cuda_ci(est_time=224, suite="stage-b-test-small-1-gpu")
+
+PROMPTS = [
+    "AI is a field of computer science focused on",
+    """
+    ### Instruction:
+    Compose a SQL query that uses the following table: users, and returns the user_id and name of all users whose name that does not have a duplicate in the table.
+    ### Response:
+    SELECT user_id, name FROM users WHERE name LIKE 'A%';
+    """,
+]
+
+ADAPTERS = [
+    "faridlazuarda/valadapt-llama-3.1-8B-it-chinese",  # target_modules = q, v
+    "philschmid/code-llama-3-1-8b-text-to-sql-lora",  # target_modules = q, k, v, o, gate, up, down
+]
+
+BASE_MODEL = "meta-llama/Meta-Llama-3.1-8B-Instruct"
+
+
+@contextlib.contextmanager
+def dynamically_loaded_adapter(runner, lora_path: str, lora_name: str):
+    """A context manager to load and automatically unload a LoRA adapter."""
+    try:
+        runner.load_lora_adapter(lora_name=lora_name, lora_path=lora_path)
+        yield
+    finally:
+        runner.unload_lora_adapter(lora_name=lora_name)
+
+
+class TestLoRAEviction(CustomTestCase):
+    def test_lora_eviction_with_different_target_modules(self):
+        """
+        Test LoRA eviction with different target modules.
+
+        This test runs inference against two LoRA adapters in different orders to force eviction behavior, and ensures
+        that the outputs of the same (adapter, prompt) pair are consistent across runs.
+        """
+        output_history = {}
+        self._run_test(ADAPTERS, output_history, reverse=False)
+        self._run_test(ADAPTERS, output_history, reverse=True)
+
+    def test_lora_eviction_with_reused_lora_name(self):
+        """
+        Test LoRA eviction with reused LoRA names.
+
+        This test runs inference against two LoRA adapters with the same name to ensure that the eviction behavior
+        works correctly when reusing LoRA names.
+        """
+        output_history = {}
+        self._run_test(ADAPTERS, output_history, reuse_lora_name=True, repeat=1)
+        self._run_test(ADAPTERS, output_history, reuse_lora_name=False, repeat=1)
+
+    def _run_test(
+        self,
+        lora_paths: List[str],
+        output_history: Dict[Tuple[str, str], str],
+        reverse: bool = False,
+        repeat: int = 2,
+        reuse_lora_name: bool = False,
+    ):
+        REUSED_LORA_NAME = "lora"
+        max_new_tokens = 256
+        torch_dtype = torch.float16
+        base_path = BASE_MODEL
+        assert len(lora_paths) >= 2
+
+        initial_lora_paths = lora_paths if not reuse_lora_name else None
+        # Initialize runners
+        with SRTRunner(
+            base_path,
+            torch_dtype=torch_dtype,
+            model_type="generation",
+            lora_paths=initial_lora_paths,
+            max_loras_per_batch=1,
+            enable_lora=True,
+            max_lora_rank=256,
+            # Need to list all lora modules, or "all" might include lora modules without assigning lora weights
+            # lora_target_modules=["all"],
+            lora_target_modules=[
+                "q_proj",
+                "k_proj",
+                "v_proj",
+                "o_proj",
+                "gate_proj",
+                "up_proj",
+                "down_proj",
+            ],
+        ) as srt_runner:
+            adapter_sequence = lora_paths if not reverse else lora_paths[::-1]
+
+            for i in range(repeat):
+                for j, lora_path in enumerate(adapter_sequence):
+                    print(
+                        f"\n========== Testing LoRA eviction with adapter '{lora_path}' (#{j + 1}/{len(adapter_sequence)}), reuse_lora_name: {reuse_lora_name}, reversed: {reverse}, repeat: {i + 1}/{repeat} ---"
+                    )
+
+                    lora_name = REUSED_LORA_NAME if reuse_lora_name else lora_path
+                    context = (
+                        dynamically_loaded_adapter(srt_runner, lora_path, lora_name)
+                        if reuse_lora_name
+                        else contextlib.nullcontext()
+                    )
+                    with context:
+                        for prompt in PROMPTS:
+                            print("\nprompt:\n", prompt)
+                            srt_outputs = srt_runner.forward(
+                                [prompt],
+                                max_new_tokens=max_new_tokens,
+                                lora_paths=[lora_name],
+                            )
+                            output = srt_outputs.output_strs[0].strip()
+                            print("\noutput:\n", output)
+
+                            prev_output = output_history.get((lora_path, prompt))
+                            if prev_output is not None:
+                                self.assertEqual(
+                                    prev_output,
+                                    output,
+                                    f"Output mismatch for adapter {lora_path} and prompt '{prompt}' on repeat {j + 1}, previous: '{prev_output}', current: '{output}'.",
+                                )
+                            else:
+                                output_history[(lora_path, prompt)] = output
+
+
+if __name__ == "__main__":
+    try:
+        mp.set_start_method("spawn")
+    except RuntimeError:
+        pass
+
+    unittest.main(warnings="ignore")

test/registered/lora/test_lora_eviction_policy.py

@@ -0,0 +1,193 @@
+# Copyright 2023-2024 SGLang Team
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+"""
+Unit tests for LoRA eviction policies.
+Tests LRU and FIFO eviction behavior.
+"""
+
+import unittest
+
+from sglang.srt.lora.eviction_policy import get_eviction_policy
+from sglang.test.ci.ci_register import register_cuda_ci
+
+register_cuda_ci(est_time=200, suite="nightly-1-gpu", nightly=True)
+
+
+class TestLoRAEvictionPolicy(unittest.TestCase):
+    """Unit tests for LoRA eviction policies."""
+
+    def _test_eviction_policy(
+        self, policy_name, access_sequence, candidates, expected_victim
+    ):
+        """
+        Helper to test eviction policy with given access pattern.
+
+        Args:
+            policy_name: Name of eviction policy ("lru" or "fifo")
+            access_sequence: List of adapter IDs in access order
+            candidates: Set of adapter IDs that can be evicted
+            expected_victim: Expected adapter ID to be evicted
+        """
+        policy = get_eviction_policy(policy_name)
+
+        # Simulate access pattern
+        for adapter_id in access_sequence:
+            policy.mark_used(adapter_id)
+
+        # Select victim from candidates
+        victim = policy.select_victim(candidates)
+        self.assertEqual(
+            victim,
+            expected_victim,
+            f"{policy_name.upper()}: Expected {expected_victim}, got {victim}",
+        )
+
+    def test_lru_basic(self):
+        """Test LRU selects least recently used adapter."""
+        self._test_eviction_policy(
+            "lru",
+            access_sequence=["lora1", "lora2", "lora3", "lora4"],
+            candidates={"lora1", "lora2", "lora3", "lora4"},
+            expected_victim="lora1",
+        )
+
+    def test_lru_with_reuse(self):
+        """Test LRU updates order on reuse."""
+        self._test_eviction_policy(
+            "lru",
+            access_sequence=["lora1", "lora2", "lora3", "lora4", "lora1"],
+            candidates={"lora1", "lora2", "lora3", "lora4"},
+            expected_victim="lora2",
+        )
+
+    def test_lru_multiple_reuse(self):
+        """Test LRU with multiple reuses."""
+        self._test_eviction_policy(
+            "lru",
+            access_sequence=["lora1", "lora2", "lora3", "lora1", "lora2"],
+            candidates={"lora1", "lora2", "lora3"},
+            expected_victim="lora3",
+        )
+
+    def test_lru_with_subset_candidates(self):
+        """Test LRU with subset of candidates."""
+        self._test_eviction_policy(
+            "lru",
+            access_sequence=["lora1", "lora2", "lora3", "lora4"],
+            candidates={"lora2", "lora3", "lora4"},
+            expected_victim="lora2",
+        )
+
+    def test_lru_base_model_evicted_last(self):
+        """Test LRU evicts LoRA adapters before base model (None)."""
+        self._test_eviction_policy(
+            "lru",
+            access_sequence=["lora1", "lora2", "lora3"],
+            candidates={None, "lora1", "lora2", "lora3"},
+            expected_victim="lora1",
+        )
+
+    def test_fifo_basic(self):
+        """Test FIFO selects first inserted adapter."""
+        self._test_eviction_policy(
+            "fifo",
+            access_sequence=["lora1", "lora2", "lora3", "lora4"],
+            candidates={"lora1", "lora2", "lora3", "lora4"},
+            expected_victim="lora1",
+        )
+
+    def test_fifo_ignores_reuse(self):
+        """Test FIFO ignores reuse."""
+        self._test_eviction_policy(
+            "fifo",
+            access_sequence=[
+                "lora1",
+                "lora2",
+                "lora3",
+                "lora4",
+                "lora4",
+                "lora3",
+                "lora2",
+                "lora1",
+            ],
+            candidates={"lora1", "lora2", "lora3", "lora4"},
+            expected_victim="lora1",
+        )
+
+    def test_fifo_with_subset_candidates(self):
+        """Test FIFO with subset of candidates."""
+        self._test_eviction_policy(
+            "fifo",
+            access_sequence=["lora1", "lora2", "lora3", "lora4"],
+            candidates={"lora2", "lora3", "lora4"},
+            expected_victim="lora2",
+        )
+
+    def test_fifo_base_model_evicted_last(self):
+        """Test FIFO evicts LoRA adapters before base model (None)."""
+        self._test_eviction_policy(
+            "fifo",
+            access_sequence=["lora1", "lora2", "lora3"],
+            candidates={None, "lora1", "lora2", "lora3"},
+            expected_victim="lora1",
+        )
+
+    def test_policy_remove(self):
+        """Test that remove() correctly updates internal state."""
+        lru = get_eviction_policy("lru")
+        lru.mark_used("lora1")
+        lru.mark_used("lora2")
+        lru.mark_used("lora3")
+
+        # Remove lora1, so lora2 becomes LRU
+        lru.remove("lora1")
+        victim = lru.select_victim({"lora1", "lora2", "lora3"})
+        self.assertEqual(victim, "lora2")
+
+    def test_eviction_policy_factory(self):
+        """Test eviction policy factory function."""
+        # Test valid policies
+        lru = get_eviction_policy("lru")
+        fifo = get_eviction_policy("fifo")
+
+        self.assertIsNotNone(lru)
+        self.assertIsNotNone(fifo)
+
+        # Test invalid policy
+        with self.assertRaises(ValueError):
+            get_eviction_policy("invalid_policy")
+
+    def test_lru_vs_fifo_behavior(self):
+        """Test that LRU and FIFO behave differently."""
+        access_sequence = ["lora1", "lora2", "lora3", "lora1"]
+        candidates = {"lora1", "lora2", "lora3"}
+
+        lru = get_eviction_policy("lru")
+        for adapter_id in access_sequence:
+            lru.mark_used(adapter_id)
+        lru_victim = lru.select_victim(candidates)
+
+        fifo = get_eviction_policy("fifo")
+        for adapter_id in access_sequence:
+            fifo.mark_used(adapter_id)
+        fifo_victim = fifo.select_victim(candidates)
+
+        self.assertNotEqual(lru_victim, fifo_victim)
+        self.assertEqual(lru_victim, "lora2")
+        self.assertEqual(fifo_victim, "lora1")
+
+
+if __name__ == "__main__":
+    unittest.main(verbosity=2)

test/registered/lora/test_lora_hf_sgl_logprob_diff.py

@@ -0,0 +1,548 @@
+# Copyright 2023-2024 SGLang Team
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+"""
+Test to compare log probabilities between HuggingFace+LoRA and SGLang+LoRA.
+
+This test:
+1. Runs SGLang with LoRA and collects log probabilities
+2. Runs HuggingFace with LoRA and collects log probabilities
+3. Compares the differences (max and mean) between the two implementations
+4. Uses unittest framework for easy integration with test suites
+
+Usage:
+    python test_lora_hf_sgl_logprob_diff.py
+    or
+    python -m unittest test_lora_hf_sgl_logprob_diff
+"""
+
+import multiprocessing as mp
+import unittest
+from typing import Any, Dict, List, Optional, Tuple
+
+import numpy as np
+import torch
+
+from sglang.test.ci.ci_register import register_cuda_ci
+from sglang.test.runners import HFRunner, SRTRunner
+
+register_cuda_ci(
+    est_time=300,
+    suite="nightly-1-gpu",
+    nightly=True,
+    disabled="Temporarily disabled, will be fixed later",
+)
+
+from sglang.test.test_utils import DEFAULT_PORT_FOR_SRT_TEST_RUNNER, CustomTestCase
+
+# Test configuration constants
+LORA_BACKEND = "triton"
+DISABLE_CUDA_GRAPH = False
+LORA_TARGET_MODULES = None
+LOGPROB_THRESHOLD = 1e-01
+
+# Default test prompts
+DEFAULT_TEST_PROMPTS = [
+    "SGL is a",
+    "AI is a field of computer science focused on",
+    "Computer science is the study of",
+    "Write a short story.",
+    "What are the main components of a computer?",
+]
+
+# Formatting constants
+DIVIDER_WIDTH = 80
+SECTION_CHAR = "="
+SUBSECTION_CHAR = "-"
+
+
+def print_section_header(title: str):
+    """Print a major section header."""
+    print("\n" + SECTION_CHAR * DIVIDER_WIDTH)
+    print(title)
+    print(SECTION_CHAR * DIVIDER_WIDTH)
+
+
+def print_subsection_header(title: str):
+    """Print a subsection header."""
+    print(f"\n{SUBSECTION_CHAR * 40}")
+    print(f"{title}")
+    print(SUBSECTION_CHAR * 40)
+
+
+def print_config_info(title: str, config: Dict[str, Any]):
+    """Print configuration information in a consistent format."""
+    print_section_header(title)
+    for key, value in config.items():
+        print(f"  {key}: {value}")
+
+
+def compare_logprobs_for_type(
+    sglang_logprobs: torch.Tensor, hf_logprobs: torch.Tensor, logprob_type: str
+) -> Dict[str, Any]:
+    """
+    Compare logprobs for a specific type (prefill or decode).
+
+    Args:
+        sglang_logprobs: SGLang log probabilities
+        hf_logprobs: HuggingFace log probabilities
+        logprob_type: Type of logprobs ("prefill" or "decode")
+
+    Returns:
+        Dictionary containing comparison statistics
+    """
+    diff = torch.abs(sglang_logprobs - hf_logprobs)
+    max_diff = torch.max(diff).item()
+    mean_diff = torch.mean(diff).item()
+    shape = list(sglang_logprobs.shape)
+    matches_threshold = max_diff < LOGPROB_THRESHOLD
+
+    return {
+        "max_diff": max_diff,
+        "mean_diff": mean_diff,
+        "shape": shape,
+        "matches_threshold": matches_threshold,
+        "type": logprob_type,
+    }
+
+
+def print_logprob_comparison(comparison: Dict[str, Any]):
+    """Print logprob comparison results in a consistent format."""
+    logprob_type = comparison["type"].capitalize()
+    print(f"\n{logprob_type} logprobs:")
+    print(f"  Shape:           {comparison['shape']}")
+    print(f"  Max difference:  {comparison['max_diff']:.6e}")
+    print(f"  Mean difference: {comparison['mean_diff']:.6e}")
+
+    status = "PASS" if comparison["matches_threshold"] else "FAIL"
+    print(f"  Status:          {status} (threshold: {LOGPROB_THRESHOLD:.0e})")
+
+
+def compare_output_strings(
+    sglang_output: str, hf_output: str, max_display_len: int = 200
+) -> Dict[str, Any]:
+    """
+    Compare output strings between SGLang and HuggingFace.
+
+    Args:
+        sglang_output: SGLang generated text
+        hf_output: HuggingFace generated text
+        max_display_len: Maximum length for display
+
+    Returns:
+        Dictionary containing comparison results
+    """
+    outputs_match = sglang_output.strip() == hf_output.strip()
+
+    # Truncate for display if needed
+    sglang_display = (
+        sglang_output[:max_display_len]
+        if len(sglang_output) > max_display_len
+        else sglang_output
+    )
+    hf_display = (
+        hf_output[:max_display_len] if len(hf_output) > max_display_len else hf_output
+    )
+
+    return {
+        "match": outputs_match,
+        "sglang_output": sglang_output,
+        "hf_output": hf_output,
+        "sglang_display": sglang_display,
+        "hf_display": hf_display,
+    }
+
+
+def print_output_comparison(comparison: Dict[str, Any]):
+    """Print output string comparison in a consistent format."""
+    print(f"\nOutput strings:")
+    status = "MATCH" if comparison["match"] else "DIFFER"
+    print(f"  Status:      {status}")
+    print(f"  SGLang:      {comparison['sglang_display']}")
+    print(f"  HuggingFace: {comparison['hf_display']}")
+
+
+def prepare_lora_paths_per_prompt(
+    lora_paths: List[str], num_prompts: int
+) -> List[Optional[str]]:
+    """
+    Prepare LoRA paths for each prompt by cycling through available LoRAs.
+
+    Args:
+        lora_paths: List of available LoRA adapter paths
+        num_prompts: Number of prompts to generate LoRA paths for
+
+    Returns:
+        List of LoRA paths (one per prompt), or None values if no LoRAs
+    """
+    if not lora_paths:
+        return [None] * num_prompts
+
+    return [lora_paths[i % len(lora_paths)] for i in range(num_prompts)]
+
+
+def run_sglang_with_lora(
+    model_path: str,
+    lora_paths: List[str],
+    prompts: List[str],
+    max_new_tokens: int,
+    torch_dtype: torch.dtype,
+    lora_backend: str,
+    port: int,
+    disable_cuda_graph: bool,
+    lora_target_modules: Optional[List[str]],
+    tp_size: int,
+) -> Dict[str, Any]:
+    """Run SGLang with LoRA and return log probabilities."""
+    config = {
+        "Model": model_path,
+        "LoRA paths": lora_paths,
+        "LoRA backend": lora_backend,
+        "Disable CUDA graph": disable_cuda_graph,
+        "Port": port,
+        "Number of prompts": len(prompts),
+        "Tensor parallel size": tp_size,
+    }
+    print_config_info("Running SGLang with LoRA", config)
+
+    lora_paths_per_prompt = prepare_lora_paths_per_prompt(lora_paths, len(prompts))
+
+    with SRTRunner(
+        model_path,
+        torch_dtype=torch_dtype,
+        model_type="generation",
+        tp_size=tp_size,
+        lora_paths=lora_paths,
+        max_loras_per_batch=len(lora_paths) if lora_paths else 1,
+        lora_backend=lora_backend,
+        disable_cuda_graph=disable_cuda_graph,
+        disable_radix_cache=True,
+        port=port,
+        mem_fraction_static=0.88,
+        lora_target_modules=lora_target_modules,
+    ) as srt_runner:
+        srt_outputs = srt_runner.forward(
+            prompts,
+            max_new_tokens=max_new_tokens,
+            lora_paths=lora_paths_per_prompt,
+        )
+
+    return {
+        "top_input_logprobs": srt_outputs.top_input_logprobs,
+        "top_output_logprobs": srt_outputs.top_output_logprobs,
+        "output_strs": srt_outputs.output_strs,
+        "lora_paths": lora_paths_per_prompt,
+    }
+
+
+def run_hf_with_lora(
+    model_path: str,
+    lora_paths: List[str],
+    prompts: List[str],
+    max_new_tokens: int,
+    torch_dtype: torch.dtype,
+) -> Dict[str, Any]:
+    """Run HuggingFace with LoRA and return log probabilities."""
+    config = {
+        "Model": model_path,
+        "LoRA paths": lora_paths,
+        "Number of prompts": len(prompts),
+    }
+    print_config_info("Running HuggingFace with LoRA", config)
+
+    lora_paths_per_prompt = prepare_lora_paths_per_prompt(lora_paths, len(prompts))
+
+    with HFRunner(
+        model_path,
+        torch_dtype=torch_dtype,
+        model_type="generation",
+        patch_model_do_sample_false=True,
+    ) as hf_runner:
+        hf_outputs = hf_runner.forward(
+            prompts,
+            max_new_tokens=max_new_tokens,
+            lora_paths=lora_paths_per_prompt,
+        )
+
+    return {
+        "top_input_logprobs": hf_outputs.top_input_logprobs,
+        "top_output_logprobs": hf_outputs.top_output_logprobs,
+        "output_strs": hf_outputs.output_strs,
+        "lora_paths": lora_paths_per_prompt,
+    }
+
+
+def compare_single_prompt(
+    prompt_idx: int,
+    sglang_data: Dict[str, Any],
+    hf_data: Dict[str, Any],
+) -> Dict[str, Any]:
+    """
+    Compare logprobs and outputs for a single prompt.
+
+    Args:
+        prompt_idx: Index of the prompt being compared
+        sglang_data: SGLang results data
+        hf_data: HuggingFace results data
+
+    Returns:
+        Dictionary containing all comparison results
+    """
+    print_subsection_header(f"Prompt {prompt_idx + 1}")
+    print(f"LoRA adapter: {sglang_data['lora_paths'][prompt_idx]}")
+
+    result = {
+        "prompt_idx": prompt_idx,
+        "lora_path": sglang_data["lora_paths"][prompt_idx],
+    }
+
+    # Compare prefill (input) logprobs
+    sglang_prefill = torch.tensor(sglang_data["top_input_logprobs"][prompt_idx])
+    hf_prefill = torch.tensor(hf_data["top_input_logprobs"][prompt_idx])
+    prefill_comparison = compare_logprobs_for_type(
+        sglang_prefill, hf_prefill, "prefill"
+    )
+    print_logprob_comparison(prefill_comparison)
+
+    # Store prefill results
+    result["prefill_max_diff"] = prefill_comparison["max_diff"]
+    result["prefill_mean_diff"] = prefill_comparison["mean_diff"]
+    result["prefill_shape"] = prefill_comparison["shape"]
+    result["prefill_logprob_match"] = prefill_comparison["matches_threshold"]
+
+    # Compare decode (output) logprobs
+    sglang_decode = torch.tensor(sglang_data["top_output_logprobs"][prompt_idx])
+    hf_decode = torch.tensor(hf_data["top_output_logprobs"][prompt_idx])
+    decode_comparison = compare_logprobs_for_type(sglang_decode, hf_decode, "decode")
+    print_logprob_comparison(decode_comparison)
+
+    # Store decode results
+    result["decode_max_diff"] = decode_comparison["max_diff"]
+    result["decode_mean_diff"] = decode_comparison["mean_diff"]
+    result["decode_shape"] = decode_comparison["shape"]
+    result["decode_logprob_match"] = decode_comparison["matches_threshold"]
+
+    # Overall logprob match
+    result["overall_logprob_match"] = (
+        prefill_comparison["matches_threshold"]
+        and decode_comparison["matches_threshold"]
+    )
+
+    # Compare output strings
+    sglang_output = sglang_data["output_strs"][prompt_idx]
+    hf_output = hf_data["output_strs"][prompt_idx]
+    output_comparison = compare_output_strings(sglang_output, hf_output)
+    print_output_comparison(output_comparison)
+
+    # Store output results
+    result["outputs_match"] = output_comparison["match"]
+    result["sglang_output"] = output_comparison["sglang_output"]
+    result["hf_output"] = output_comparison["hf_output"]
+
+    return result
+
+
+def print_overall_statistics(results: List[Dict[str, Any]]):
+    """Print overall statistics across all prompts."""
+    print_section_header("Overall Statistics")
+
+    # Gather statistics
+    prefill_max_diffs = [r["prefill_max_diff"] for r in results]
+    prefill_mean_diffs = [r["prefill_mean_diff"] for r in results]
+    decode_max_diffs = [r["decode_max_diff"] for r in results]
+    decode_mean_diffs = [r["decode_mean_diff"] for r in results]
+
+    # Print logprob statistics
+    print("\nLogprob Differences:")
+    print(f"  Prefill:")
+    print(f"    Max of max:   {max(prefill_max_diffs):.6e}")
+    print(f"    Mean of max:  {np.mean(prefill_max_diffs):.6e}")
+    print(f"    Mean of mean: {np.mean(prefill_mean_diffs):.6e}")
+
+    print(f"  Decode:")
+    print(f"    Max of max:   {max(decode_max_diffs):.6e}")
+    print(f"    Mean of max:  {np.mean(decode_max_diffs):.6e}")
+    print(f"    Mean of mean: {np.mean(decode_mean_diffs):.6e}")
+
+    # Print match statistics
+    num_prompts = len(results)
+    logprob_match_count = sum(r["overall_logprob_match"] for r in results)
+    prefill_match_count = sum(r["prefill_logprob_match"] for r in results)
+    decode_match_count = sum(r["decode_logprob_match"] for r in results)
+    outputs_match_count = sum(r["outputs_match"] for r in results)
+
+    print(f"\nLogprob Statistics (threshold: {LOGPROB_THRESHOLD:.0e}):")
+    overall_status = "PASSED" if logprob_match_count == num_prompts else "FAILED"
+    print(f"  Overall logprob: {logprob_match_count}/{num_prompts} {overall_status}")
+    print(f"  Prefill logprob: {prefill_match_count}/{num_prompts}")
+    print(f"  Decode logprob:  {decode_match_count}/{num_prompts}")
+
+    print(f"\nString Statistics:")
+    print(f"  Output strings:  {outputs_match_count}/{num_prompts}")
+
+    # Return overall stats for saving
+    return {
+        "logprob_differences": {
+            "prefill": {
+                "max_of_max_diffs": max(prefill_max_diffs),
+                "mean_of_max_diffs": float(np.mean(prefill_max_diffs)),
+                "mean_of_mean_diffs": float(np.mean(prefill_mean_diffs)),
+            },
+            "decode": {
+                "max_of_max_diffs": max(decode_max_diffs),
+                "mean_of_max_diffs": float(np.mean(decode_max_diffs)),
+                "mean_of_mean_diffs": float(np.mean(decode_mean_diffs)),
+            },
+        },
+        "match_statistics": {
+            "overall_logprob_match_rate": logprob_match_count / num_prompts,
+            "prefill_logprob_match_rate": prefill_match_count / num_prompts,
+            "decode_logprob_match_rate": decode_match_count / num_prompts,
+            "outputs_match_rate": outputs_match_count / num_prompts,
+        },
+    }
+
+
+def compare_logprobs(
+    sglang_logprobs: Dict[str, Any], hf_logprobs: Dict[str, Any]
+) -> Tuple[List[Dict[str, Any]], Dict[str, Any]]:
+    """Compare log probabilities and compute statistics."""
+    print_section_header("Comparing Log Probabilities")
+
+    results = []
+    num_prompts = len(sglang_logprobs["top_input_logprobs"])
+
+    for i in range(num_prompts):
+        result = compare_single_prompt(i, sglang_logprobs, hf_logprobs)
+        results.append(result)
+
+    overall_stats = print_overall_statistics(results)
+
+    return results, overall_stats
+
+
+class TestLoRAHFSGLLogprobDifference(CustomTestCase):
+    """
+    Test case to compare log probabilities between HuggingFace+LoRA and SGLang+LoRA.
+    """
+
+    def _run_comparison_test(
+        self,
+        model_path: str,
+        lora_paths: List[str],
+        prompts: List[str],
+        max_new_tokens: int = 32,
+        torch_dtype: torch.dtype = torch.float16,
+        lora_backend: str = LORA_BACKEND,
+        port: int = DEFAULT_PORT_FOR_SRT_TEST_RUNNER,
+        disable_cuda_graph: bool = DISABLE_CUDA_GRAPH,
+        lora_target_modules: Optional[List[str]] = LORA_TARGET_MODULES,
+        tp_size: int = 1,
+    ):
+        """
+        Run comparison test between SGLang and HuggingFace with LoRA.
+        """
+        print_section_header(f"Testing {model_path} with LoRA adapters")
+
+        # Step 1: Run SGLang with LoRA
+        sglang_logprobs = run_sglang_with_lora(
+            model_path=model_path,
+            lora_paths=lora_paths,
+            prompts=prompts,
+            max_new_tokens=max_new_tokens,
+            torch_dtype=torch_dtype,
+            lora_backend=lora_backend,
+            port=port,
+            disable_cuda_graph=disable_cuda_graph,
+            lora_target_modules=lora_target_modules,
+            tp_size=tp_size,
+        )
+
+        # Clear GPU memory
+        print("\nClearing GPU memory...")
+        torch.cuda.empty_cache()
+
+        # Step 2: Run HuggingFace with LoRA
+        hf_logprobs = run_hf_with_lora(
+            model_path=model_path,
+            lora_paths=lora_paths,
+            prompts=prompts,
+            max_new_tokens=max_new_tokens,
+            torch_dtype=torch_dtype,
+        )
+
+        # Step 3: Compare log probabilities
+        results, overall_stats = compare_logprobs(sglang_logprobs, hf_logprobs)
+
+        # Assert that all prompts pass the threshold
+        for result in results:
+            self.assertTrue(
+                result["prefill_logprob_match"],
+                f"Prefill logprob mismatch for prompt {result['prompt_idx']} "
+                f"(max_diff={result['prefill_max_diff']:.6e}, threshold={LOGPROB_THRESHOLD:.0e})",
+            )
+            self.assertTrue(
+                result["decode_logprob_match"],
+                f"Decode logprob mismatch for prompt {result['prompt_idx']} "
+                f"(max_diff={result['decode_max_diff']:.6e}, threshold={LOGPROB_THRESHOLD:.0e})",
+            )
+
+        print_section_header("Test completed successfully!")
+
+        return results, overall_stats
+
+    def test_lora_logprob_comparison_basic(self):
+        """
+        Basic test comparing HF and SGLang LoRA logprobs with small model.
+        """
+        model_path = "meta-llama/Llama-2-7b-hf"
+        lora_paths = ["yushengsu/sglang_lora_logprob_diff_without_tuning"]
+        prompts = DEFAULT_TEST_PROMPTS[:2]  # Use fewer prompts for faster testing
+
+        self._run_comparison_test(
+            model_path=model_path,
+            lora_paths=lora_paths,
+            prompts=prompts,
+            max_new_tokens=32,
+        )
+
+    def test_lora_logprob_comparison_full(self):
+        """
+        Full test comparing HF and SGLang LoRA logprobs with all prompts.
+        """
+        model_path = "meta-llama/Llama-2-7b-hf"
+        lora_paths = ["yushengsu/sglang_lora_logprob_diff_without_tuning"]
+        prompts = DEFAULT_TEST_PROMPTS
+
+        self._run_comparison_test(
+            model_path=model_path,
+            lora_paths=lora_paths,
+            prompts=prompts,
+            max_new_tokens=32,
+        )
+
+
+if __name__ == "__main__":
+    try:
+        mp.set_start_method("spawn")
+    except RuntimeError:
+        pass
+
+    try:
+        unittest.main(warnings="ignore", verbosity=2)
+    finally:
+        # Final cleanup
+        if torch.cuda.is_available():
+            torch.cuda.empty_cache()
+            torch.cuda.synchronize()

test/registered/lora/test_lora_openai_api.py

@@ -0,0 +1,330 @@
+"""
+Unit tests for OpenAI-compatible LoRA API support.
+
+Tests the model parameter parsing and LoRA adapter resolution logic
+that enables OpenAI-compatible LoRA adapter selection.
+"""
+
+import unittest
+from unittest.mock import MagicMock
+
+from sglang.srt.entrypoints.openai.serving_base import OpenAIServingBase
+from sglang.test.ci.ci_register import register_cuda_ci
+
+register_cuda_ci(est_time=30, suite="nightly-1-gpu", nightly=True)
+from sglang.srt.server_args import ServerArgs
+
+
+class MockTokenizerManager:
+    """Mock TokenizerManager for testing."""
+
+    def __init__(self, enable_lora=False):
+        self.server_args = MagicMock(spec=ServerArgs)
+        self.server_args.enable_lora = enable_lora
+        self.server_args.tokenizer_metrics_allowed_custom_labels = None
+
+
+class ConcreteServingBase(OpenAIServingBase):
+    """Concrete implementation for testing abstract base class."""
+
+    def _request_id_prefix(self) -> str:
+        return "test-"
+
+    def _convert_to_internal_request(self, request, raw_request=None):
+        pass
+
+    def _validate_request(self, request):
+        pass
+
+
+class TestParseModelParameter(unittest.TestCase):
+    """Test _parse_model_parameter method."""
+
+    def setUp(self):
+        self.tokenizer_manager = MockTokenizerManager(enable_lora=True)
+        self.serving = ConcreteServingBase(self.tokenizer_manager)
+
+    def test_model_without_adapter(self):
+        """Test parsing model without adapter returns None for adapter."""
+        base_model, adapter = self.serving._parse_model_parameter("llama-3.1-8B")
+        self.assertEqual(base_model, "llama-3.1-8B")
+        self.assertIsNone(adapter)
+
+    def test_model_with_adapter(self):
+        """Test parsing model with adapter extracts both parts."""
+        base_model, adapter = self.serving._parse_model_parameter(
+            "llama-3.1-8B:sql-expert"
+        )
+        self.assertEqual(base_model, "llama-3.1-8B")
+        self.assertEqual(adapter, "sql-expert")
+
+    def test_model_with_path_and_adapter(self):
+        """Test parsing model path with slashes and adapter."""
+        base_model, adapter = self.serving._parse_model_parameter(
+            "meta-llama/Llama-3.1-8B-Instruct:adapter-name"
+        )
+        self.assertEqual(base_model, "meta-llama/Llama-3.1-8B-Instruct")
+        self.assertEqual(adapter, "adapter-name")
+
+    def test_model_with_multiple_colons(self):
+        """Test that only first colon is used for splitting."""
+        base_model, adapter = self.serving._parse_model_parameter("model:adapter:extra")
+        self.assertEqual(base_model, "model")
+        self.assertEqual(adapter, "adapter:extra")
+
+    def test_model_with_whitespace(self):
+        """Test that whitespace is stripped from both parts."""
+        base_model, adapter = self.serving._parse_model_parameter(
+            " model-name : adapter-name "
+        )
+        self.assertEqual(base_model, "model-name")
+        self.assertEqual(adapter, "adapter-name")
+
+    def test_model_with_empty_adapter(self):
+        """Test model ending with colon returns None for adapter."""
+        base_model, adapter = self.serving._parse_model_parameter("model-name:")
+        self.assertEqual(base_model, "model-name")
+        self.assertIsNone(adapter)
+
+    def test_model_with_only_spaces_after_colon(self):
+        """Test model with only whitespace after colon returns None for adapter."""
+        base_model, adapter = self.serving._parse_model_parameter("model-name:   ")
+        self.assertEqual(base_model, "model-name")
+        self.assertIsNone(adapter)
+
+
+class TestResolveLoraPath(unittest.TestCase):
+    """Test _resolve_lora_path method."""
+
+    def setUp(self):
+        self.tokenizer_manager = MockTokenizerManager(enable_lora=True)
+        self.serving = ConcreteServingBase(self.tokenizer_manager)
+
+    def test_no_adapter_specified(self):
+        """Test when neither model nor explicit lora_path has adapter."""
+        result = self.serving._resolve_lora_path("model-name", None)
+        self.assertIsNone(result)
+
+    def test_adapter_in_model_only(self):
+        """Test adapter from model parameter when no explicit path."""
+        result = self.serving._resolve_lora_path("model:sql-expert", None)
+        self.assertEqual(result, "sql-expert")
+
+    def test_adapter_in_explicit_only(self):
+        """Test adapter from explicit lora_path when not in model."""
+        result = self.serving._resolve_lora_path("model-name", "python-expert")
+        self.assertEqual(result, "python-expert")
+
+    def test_model_parameter_takes_precedence(self):
+        """Test model parameter adapter takes precedence over explicit."""
+        result = self.serving._resolve_lora_path("model:sql-expert", "python-expert")
+        self.assertEqual(result, "sql-expert")
+
+    def test_with_list_explicit_lora_path(self):
+        """Test that explicit list is returned when no model adapter."""
+        explicit = ["adapter1", "adapter2", None]
+        result = self.serving._resolve_lora_path("model-name", explicit)
+        self.assertEqual(result, explicit)
+
+    def test_model_adapter_overrides_list(self):
+        """Test model adapter overrides even when explicit is a list."""
+        result = self.serving._resolve_lora_path(
+            "model:sql-expert", ["adapter1", "adapter2"]
+        )
+        self.assertEqual(result, "sql-expert")
+
+    def test_complex_model_name_with_adapter(self):
+        """Test resolution with complex model name."""
+        result = self.serving._resolve_lora_path(
+            "org/model-v2.1:adapter-name", "other-adapter"
+        )
+        self.assertEqual(result, "adapter-name")
+
+
+class TestValidateLoraEnabled(unittest.TestCase):
+    """Test _validate_lora_enabled method."""
+
+    def test_validation_passes_when_lora_enabled(self):
+        """Test validation passes when LoRA is enabled."""
+        tokenizer_manager = MockTokenizerManager(enable_lora=True)
+        serving = ConcreteServingBase(tokenizer_manager)
+
+        # Should not raise
+        try:
+            serving._validate_lora_enabled("sql-expert")
+        except ValueError:
+            self.fail("_validate_lora_enabled raised ValueError unexpectedly")
+
+    def test_validation_fails_when_lora_disabled(self):
+        """Test validation fails with helpful message when LoRA is disabled."""
+        tokenizer_manager = MockTokenizerManager(enable_lora=False)
+        serving = ConcreteServingBase(tokenizer_manager)
+
+        with self.assertRaises(ValueError) as context:
+            serving._validate_lora_enabled("sql-expert")
+
+        error_message = str(context.exception)
+        self.assertIn("sql-expert", error_message)
+        self.assertIn("--enable-lora", error_message)
+        self.assertIn("not enabled", error_message)
+
+    def test_validation_error_mentions_adapter_name(self):
+        """Test that error message includes the requested adapter name."""
+        tokenizer_manager = MockTokenizerManager(enable_lora=False)
+        serving = ConcreteServingBase(tokenizer_manager)
+
+        with self.assertRaises(ValueError) as context:
+            serving._validate_lora_enabled("my-custom-adapter")
+
+        self.assertIn("my-custom-adapter", str(context.exception))
+
+
+class TestIntegrationScenarios(unittest.TestCase):
+    """Integration tests for common usage scenarios."""
+
+    def setUp(self):
+        self.tokenizer_manager = MockTokenizerManager(enable_lora=True)
+        self.serving = ConcreteServingBase(self.tokenizer_manager)
+
+    def test_openai_compatible_usage(self):
+        """Test typical OpenAI-compatible usage pattern."""
+        # User specifies adapter in model parameter
+        model = "meta-llama/Llama-3.1-8B:sql-expert"
+        explicit_lora = None
+
+        lora_path = self.serving._resolve_lora_path(model, explicit_lora)
+        self.assertEqual(lora_path, "sql-expert")
+
+        # Validation should pass
+        self.serving._validate_lora_enabled(lora_path)
+
+    def test_backward_compatible_usage(self):
+        """Test backward-compatible usage with explicit lora_path."""
+        model = "meta-llama/Llama-3.1-8B"
+        explicit_lora = "sql-expert"
+
+        lora_path = self.serving._resolve_lora_path(model, explicit_lora)
+        self.assertEqual(lora_path, "sql-expert")
+
+        # Validation should pass
+        self.serving._validate_lora_enabled(lora_path)
+
+    def test_base_model_usage(self):
+        """Test using base model without any adapter."""
+        model = "meta-llama/Llama-3.1-8B"
+        explicit_lora = None
+
+        lora_path = self.serving._resolve_lora_path(model, explicit_lora)
+        self.assertIsNone(lora_path)
+
+        # No validation needed when no adapter
+
+    def test_batch_request_scenario(self):
+        """Test batch request with list of adapters."""
+        model = "meta-llama/Llama-3.1-8B"  # No adapter in model
+        explicit_lora = ["sql-expert", "python-expert", None]
+
+        lora_path = self.serving._resolve_lora_path(model, explicit_lora)
+        self.assertEqual(lora_path, explicit_lora)
+
+        # Validate first adapter in list
+        if isinstance(lora_path, list) and lora_path[0]:
+            self.serving._validate_lora_enabled(lora_path[0])
+
+    def test_adapter_in_model_overrides_batch_list(self):
+        """Test that adapter in model parameter overrides batch list."""
+        model = "meta-llama/Llama-3.1-8B:preferred-adapter"
+        explicit_lora = ["adapter1", "adapter2"]
+
+        lora_path = self.serving._resolve_lora_path(model, explicit_lora)
+        self.assertEqual(lora_path, "preferred-adapter")
+
+    def test_error_when_lora_not_enabled(self):
+        """Test comprehensive error flow when LoRA is not enabled."""
+        # Setup server without LoRA enabled
+        tokenizer_manager = MockTokenizerManager(enable_lora=False)
+        serving = ConcreteServingBase(tokenizer_manager)
+
+        # User tries to use adapter
+        model = "meta-llama/Llama-3.1-8B:sql-expert"
+        lora_path = serving._resolve_lora_path(model, None)
+
+        # Should get helpful error
+        with self.assertRaises(ValueError) as context:
+            serving._validate_lora_enabled(lora_path)
+
+        error = str(context.exception)
+        self.assertIn("--enable-lora", error)
+        self.assertIn("sql-expert", error)
+
+
+class TestEdgeCases(unittest.TestCase):
+    """Test edge cases and error conditions."""
+
+    def setUp(self):
+        self.tokenizer_manager = MockTokenizerManager(enable_lora=True)
+        self.serving = ConcreteServingBase(self.tokenizer_manager)
+
+    def test_empty_string_model(self):
+        """Test handling of empty string model."""
+        base, adapter = self.serving._parse_model_parameter("")
+        self.assertEqual(base, "")
+        self.assertIsNone(adapter)
+
+    def test_only_colon(self):
+        """Test model parameter that is just a colon."""
+        base, adapter = self.serving._parse_model_parameter(":")
+        self.assertEqual(base, "")
+        self.assertIsNone(adapter)
+
+    def test_empty_list_lora_path(self):
+        """Test validation with empty list doesn't crash."""
+        lora_path = self.serving._resolve_lora_path("model-name", [])
+        # Empty list is falsy, so validation won't be called
+        self.assertEqual(lora_path, [])
+
+    def test_list_with_none_first(self):
+        """Test validation finds first non-None adapter in list."""
+        lora_path = self.serving._resolve_lora_path("model-name", [None, "adapter2"])
+        self.assertEqual(lora_path, [None, "adapter2"])
+        # In actual usage, validation would find "adapter2"
+
+    def test_list_all_none(self):
+        """Test validation with list of all None values."""
+        lora_path = self.serving._resolve_lora_path("model-name", [None, None])
+        self.assertEqual(lora_path, [None, None])
+        # In actual usage, no validation would occur (no non-None adapters)
+
+    def test_unicode_in_adapter_name(self):
+        """Test Unicode characters in adapter name."""
+        base, adapter = self.serving._parse_model_parameter("model:adapter-名前")
+        self.assertEqual(base, "model")
+        self.assertEqual(adapter, "adapter-名前")
+
+    def test_special_characters_in_adapter(self):
+        """Test special characters in adapter name."""
+        base, adapter = self.serving._parse_model_parameter("model:adapter_v2.1-final")
+        self.assertEqual(base, "model")
+        self.assertEqual(adapter, "adapter_v2.1-final")
+
+    def test_none_as_explicit_lora_path(self):
+        """Test None as explicit lora_path is handled correctly."""
+        result = self.serving._resolve_lora_path("model:adapter", None)
+        self.assertEqual(result, "adapter")
+
+    def test_empty_string_as_explicit_lora_path(self):
+        """Test empty string as explicit lora_path."""
+        result = self.serving._resolve_lora_path("model-name", "")
+        self.assertEqual(result, "")
+
+    def test_validation_with_empty_adapter_name(self):
+        """Test validation with empty adapter name still raises error."""
+        tokenizer_manager = MockTokenizerManager(enable_lora=False)
+        serving = ConcreteServingBase(tokenizer_manager)
+
+        with self.assertRaises(ValueError):
+            serving._validate_lora_enabled("")
+
+
+if __name__ == "__main__":
+    unittest.main()

test/registered/lora/test_lora_openai_compatible.py

@@ -0,0 +1,282 @@
+"""
+End-to-end tests for OpenAI-compatible LoRA adapter usage.
+
+Tests the model:adapter syntax and backward compatibility with explicit lora_path.
+
+Usage:
+    python3 -m unittest openai_server.features.test_lora_openai_compatible.TestLoRAOpenAICompatible.test_model_adapter_syntax
+    python3 -m unittest openai_server.features.test_lora_openai_compatible.TestLoRAOpenAICompatible.test_explicit_lora_path
+    python3 -m unittest openai_server.features.test_lora_openai_compatible.TestLoRAOpenAICompatible.test_priority_model_over_explicit
+    python3 -m unittest openai_server.features.test_lora_openai_compatible.TestLoRAOpenAICompatible.test_base_model_no_adapter
+    python3 -m unittest openai_server.features.test_lora_openai_compatible.TestLoRAOpenAICompatible.test_completions_api_with_adapter
+    python3 -m unittest openai_server.features.test_lora_openai_compatible.TestLoRAOpenAICompatible.test_streaming_with_adapter
+    python3 -m unittest openai_server.features.test_lora_openai_compatible.TestLoRADisabledError.test_lora_disabled_error
+"""
+
+import unittest
+
+import openai
+
+from sglang.test.ci.ci_register import register_cuda_ci
+
+register_cuda_ci(est_time=150, suite="nightly-1-gpu", nightly=True)
+
+from sglang.srt.utils import kill_process_tree
+from sglang.test.test_utils import (
+    DEFAULT_SMALL_MODEL_NAME_FOR_TEST,
+    DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
+    DEFAULT_URL_FOR_TEST,
+    CustomTestCase,
+    popen_launch_server,
+)
+
+
+def get_real_lora_adapter() -> str:
+    """Use a real LoRA adapter from Hugging Face."""
+    return "codelion/Llama-3.2-1B-Instruct-tool-calling-lora"
+
+
+def setup_class(cls, enable_lora=True):
+    """Setup test class with LoRA-enabled server."""
+    cls.model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST
+    cls.base_url = DEFAULT_URL_FOR_TEST
+
+    # Use real LoRA adapter
+    cls.lora_adapter_path = get_real_lora_adapter()
+
+    other_args = [
+        "--max-running-requests",
+        "10",
+        "--disable-radix-cache",  # Disable cache for cleaner tests
+    ]
+
+    if enable_lora:
+        other_args.extend(
+            [
+                "--enable-lora",
+                "--lora-paths",
+                f"tool_calling={cls.lora_adapter_path}",
+            ]
+        )
+
+    cls.process = popen_launch_server(
+        cls.model,
+        cls.base_url,
+        timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
+        other_args=other_args,
+    )
+    cls.client = openai.Client(api_key="EMPTY", base_url=f"{cls.base_url}/v1")
+
+
+class TestLoRAOpenAICompatible(CustomTestCase):
+    """Test OpenAI-compatible LoRA adapter usage."""
+
+    @classmethod
+    def setUpClass(cls):
+        setup_class(cls, enable_lora=True)
+
+    @classmethod
+    def tearDownClass(cls):
+        kill_process_tree(cls.process.pid)
+
+    def test_model_adapter_syntax(self):
+        """Test the new model:adapter syntax works correctly."""
+        response = self.client.chat.completions.create(
+            # ← New OpenAI-compatible syntax
+            model=f"{self.model}:tool_calling",
+            messages=[{"role": "user", "content": "What tools do you have available?"}],
+            max_tokens=50,
+            temperature=0,
+        )
+
+        self.assertIsNotNone(response.choices[0].message.content)
+        self.assertGreater(len(response.choices[0].message.content), 0)
+        print(f"Model adapter syntax response: {response.choices[0].message.content}")
+
+    def test_explicit_lora_path(self):
+        """Test backward compatibility with explicit lora_path via extra_body."""
+        response = self.client.chat.completions.create(
+            model=self.model,
+            messages=[{"role": "user", "content": "What tools do you have available?"}],
+            # ← Legacy explicit method
+            extra_body={"lora_path": "tool_calling"},
+            max_tokens=50,
+            temperature=0,
+        )
+
+        self.assertIsNotNone(response.choices[0].message.content)
+        self.assertGreater(len(response.choices[0].message.content), 0)
+        print(f"Explicit lora_path response: {response.choices[0].message.content}")
+
+    def test_priority_model_over_explicit(self):
+        """Test that model:adapter syntax takes precedence over explicit lora_path."""
+        # This test verifies the priority logic in _resolve_lora_path
+        response = self.client.chat.completions.create(
+            # ← Model specifies tool_calling adapter
+            model=f"{self.model}:tool_calling",
+            messages=[{"role": "user", "content": "What tools do you have available?"}],
+            # ← Both specify same adapter
+            extra_body={"lora_path": "tool_calling"},
+            max_tokens=50,
+            temperature=0,
+        )
+
+        # Should use tool_calling adapter (model parameter takes precedence)
+        self.assertIsNotNone(response.choices[0].message.content)
+        self.assertGreater(len(response.choices[0].message.content), 0)
+        print(f"Priority test response: {response.choices[0].message.content}")
+
+    def test_base_model_no_adapter(self):
+        """Test using base model without any adapter."""
+        response = self.client.chat.completions.create(
+            model=self.model,  # ← No adapter specified
+            messages=[{"role": "user", "content": "Hello, how are you?"}],
+            max_tokens=30,
+            temperature=0,
+        )
+
+        self.assertIsNotNone(response.choices[0].message.content)
+        self.assertGreater(len(response.choices[0].message.content), 0)
+        print(f"Base model response: {response.choices[0].message.content}")
+
+    def test_completions_api_with_adapter(self):
+        """Test completions API with LoRA adapter."""
+        response = self.client.completions.create(
+            model=f"{self.model}:tool_calling",  # ← Using model:adapter syntax
+            prompt="What tools do you have available?",
+            max_tokens=50,
+            temperature=0,
+        )
+
+        self.assertIsNotNone(response.choices[0].text)
+        self.assertGreater(len(response.choices[0].text), 0)
+        print(f"Completions API response: {response.choices[0].text}")
+
+    def test_streaming_with_adapter(self):
+        """Test streaming with LoRA adapter."""
+        stream = self.client.chat.completions.create(
+            model=f"{self.model}:tool_calling",
+            messages=[{"role": "user", "content": "What tools do you have available?"}],
+            max_tokens=50,
+            temperature=0,
+            stream=True,
+        )
+
+        collected_content = ""
+        for chunk in stream:
+            if chunk.choices[0].delta.content:
+                collected_content += chunk.choices[0].delta.content
+
+        self.assertGreater(len(collected_content), 0)
+        print(f"Streaming response: {collected_content}")
+
+    def test_multiple_adapters(self):
+        """Test using different adapters in sequence."""
+        # Test tool_calling adapter
+        tool_response = self.client.chat.completions.create(
+            model=f"{self.model}:tool_calling",
+            messages=[{"role": "user", "content": "What tools do you have available?"}],
+            max_tokens=30,
+            temperature=0,
+        )
+
+        # Test base model without adapter
+        base_response = self.client.chat.completions.create(
+            model=self.model,
+            messages=[{"role": "user", "content": "Hello, how are you?"}],
+            max_tokens=30,
+            temperature=0,
+        )
+
+        self.assertIsNotNone(tool_response.choices[0].message.content)
+        self.assertIsNotNone(base_response.choices[0].message.content)
+        print(
+            f"Tool calling adapter response: {tool_response.choices[0].message.content}"
+        )
+        print(f"Base model response: {base_response.choices[0].message.content}")
+
+
+class TestLoRADisabledError(CustomTestCase):
+    """Test error handling when LoRA is disabled."""
+
+    @classmethod
+    def setUpClass(cls):
+        setup_class(cls, enable_lora=False)  # ← LoRA disabled
+
+    @classmethod
+    def tearDownClass(cls):
+        kill_process_tree(cls.process.pid)
+
+    def test_lora_disabled_error(self):
+        """Test that using LoRA adapter when LoRA is disabled raises appropriate error."""
+        with self.assertRaises(openai.APIError) as context:
+            self.client.chat.completions.create(
+                model=f"{self.model}:tool_calling",  # ← Trying to use adapter
+                messages=[
+                    {"role": "user", "content": "What tools do you have available?"}
+                ],
+                max_tokens=50,
+            )
+
+        # Verify the error message contains helpful guidance
+        error_message = str(context.exception)
+        self.assertIn("LoRA", error_message)
+        self.assertIn("not enabled", error_message)
+        print(f"Expected error message: {error_message}")
+
+
+class TestLoRAEdgeCases(CustomTestCase):
+    """Test edge cases for LoRA adapter usage."""
+
+    @classmethod
+    def setUpClass(cls):
+        setup_class(cls, enable_lora=True)
+
+    @classmethod
+    def tearDownClass(cls):
+        kill_process_tree(cls.process.pid)
+
+    def test_model_with_colon_no_adapter(self):
+        """Test model parameter ending with colon (empty adapter)."""
+        response = self.client.chat.completions.create(
+            model=f"{self.model}:",  # ← Model ends with colon
+            messages=[{"role": "user", "content": "Hello!"}],
+            max_tokens=30,
+            temperature=0,
+        )
+
+        # Should work as base model (no adapter)
+        self.assertIsNotNone(response.choices[0].message.content)
+        print(f"Model with colon response: {response.choices[0].message.content}")
+
+    def test_explicit_lora_path_none(self):
+        """Test explicit lora_path set to None."""
+        response = self.client.chat.completions.create(
+            model=self.model,
+            messages=[{"role": "user", "content": "Hello!"}],
+            extra_body={"lora_path": None},  # ← Explicitly None
+            max_tokens=30,
+            temperature=0,
+        )
+
+        # Should work as base model
+        self.assertIsNotNone(response.choices[0].message.content)
+        print(
+            f"Explicit None lora_path response: {response.choices[0].message.content}"
+        )
+
+    def test_invalid_adapter_name(self):
+        """Test using non-existent adapter name."""
+        with self.assertRaises(openai.APIError) as context:
+            self.client.chat.completions.create(
+                model=f"{self.model}:nonexistent",  # ← Non-existent adapter
+                messages=[{"role": "user", "content": "Hello!"}],
+                max_tokens=30,
+            )
+
+        error_message = str(context.exception)
+        print(f"Invalid adapter error: {error_message}")
+
+
+if __name__ == "__main__":
+    unittest.main()

test/registered/lora/test_lora_qwen3.py

@@ -0,0 +1,58 @@
+# Copyright 2023-2025 SGLang Team
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+import multiprocessing as mp
+import unittest
+
+from sglang.test.ci.ci_register import register_cuda_ci
+from sglang.test.lora_utils import (
+    LoRAAdaptor,
+    LoRAModelCase,
+    run_lora_multiple_batch_on_model_cases,
+)
+
+register_cuda_ci(est_time=97, suite="nightly-1-gpu", nightly=True)
+
+from sglang.test.test_utils import CustomTestCase
+
+LORA_MODELS_QWEN3 = [
+    LoRAModelCase(
+        base="Qwen/Qwen3-4B",
+        adaptors=[
+            LoRAAdaptor(
+                name="nissenj/Qwen3-4B-lora-v2",
+                prefill_tolerance=3e-1,
+            ),
+            LoRAAdaptor(
+                name="y9760210/Qwen3-4B-lora_model",
+                prefill_tolerance=3e-1,
+            ),
+        ],
+        max_loras_per_batch=2,
+    ),
+]
+
+
+class TestLoRAQwen3(CustomTestCase):
+    def test_ci_lora_models(self):
+        run_lora_multiple_batch_on_model_cases(LORA_MODELS_QWEN3)
+
+
+if __name__ == "__main__":
+    try:
+        mp.set_start_method("spawn")
+    except RuntimeError:
+        pass
+
+    unittest.main(warnings="ignore")

test/registered/lora/test_lora_radix_cache.py

@@ -0,0 +1,82 @@
+# Copyright 2023-2024 SGLang Team
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+import multiprocessing as mp
+import unittest
+
+import torch
+
+from sglang.test.ci.ci_register import register_cuda_ci
+from sglang.test.lora_utils import CI_MULTI_LORA_MODELS, run_lora_test_one_by_one
+
+register_cuda_ci(est_time=200, suite="nightly-1-gpu", nightly=True)
+
+from sglang.test.test_utils import CustomTestCase
+
+PROMPTS = [
+    "AI is a field of computer science focused on",
+    """
+    ### Instruction:
+    Tell me about llamas and alpacas
+    ### Response:
+    Llamas are large, long-necked animals with a woolly coat. They have two toes on each foot instead of three like other camelids.
+    ### Question:
+    What do you know about llamas?
+    ### Answer:
+    """,
+]
+
+
+class TestLoRARadixCache(CustomTestCase):
+
+    def test_lora_radix_cache(self):
+        # Here we need a model case with multiple adaptors for testing correctness of radix cache
+        model_case = CI_MULTI_LORA_MODELS[0]
+
+        torch_dtype = torch.float16
+        max_new_tokens = 32
+        batch_prompts = (
+            PROMPTS
+            if not model_case.skip_long_prompt
+            else [p for p in PROMPTS if len(p) < 1000]
+        )
+
+        # Test lora with radix cache
+        run_lora_test_one_by_one(
+            batch_prompts,
+            model_case,
+            torch_dtype,
+            max_new_tokens=max_new_tokens,
+            disable_radix_cache=False,
+            test_tag="lora-with-radix-cache",
+        )
+
+        # Test lora without radix cache
+        run_lora_test_one_by_one(
+            batch_prompts,
+            model_case,
+            torch_dtype,
+            max_new_tokens=max_new_tokens,
+            disable_radix_cache=True,
+            test_tag="lora-without-radix-cache",
+        )
+
+
+if __name__ == "__main__":
+    try:
+        mp.set_start_method("spawn")
+    except RuntimeError:
+        pass
+
+    unittest.main(warnings="ignore")

test/registered/lora/test_lora_tp.py

@@ -0,0 +1,79 @@
+# Copyright 2023-2024 SGLang Team
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+import multiprocessing as mp
+import os
+import unittest
+from typing import List
+
+from sglang.test.ci.ci_register import register_cuda_ci
+from sglang.test.lora_utils import (
+    ALL_OTHER_LORA_MODELS,
+    CI_LORA_MODELS,
+    DEFAULT_PROMPTS,
+    TORCH_DTYPES,
+    LoRAModelCase,
+    run_lora_test_one_by_one,
+)
+from sglang.test.test_utils import CustomTestCase, is_in_ci
+
+register_cuda_ci(est_time=116, suite="stage-b-test-small-2-gpu")
+
+
+class TestLoRATP(CustomTestCase):
+
+    def _run_tp_on_model_cases(self, model_cases: List[LoRAModelCase]):
+        tp_list = [2]  # Define TP sizes to iterate over
+        for model_case in model_cases:
+            # If skip_long_prompt is True, filter out prompts longer than 1000 characters
+            prompts = (
+                DEFAULT_PROMPTS
+                if not model_case.skip_long_prompt
+                else [p for p in DEFAULT_PROMPTS if len(p) < 1000]
+            )
+            for tp_size in tp_list:
+                model_case.tp_size = tp_size
+                for torch_dtype in TORCH_DTYPES:
+                    run_lora_test_one_by_one(
+                        prompts,
+                        model_case,
+                        torch_dtype,
+                        max_new_tokens=32,
+                        test_tag=f"tp={tp_size}",
+                    )
+
+    def test_ci_lora_models(self):
+        self._run_tp_on_model_cases(CI_LORA_MODELS)
+
+    def test_all_lora_models(self):
+        if is_in_ci():
+            return
+
+        # Retain ONLY_RUN check here
+        filtered_models = []
+        for model_case in ALL_OTHER_LORA_MODELS:
+            if "ONLY_RUN" in os.environ and os.environ["ONLY_RUN"] != model_case.base:
+                continue
+            filtered_models.append(model_case)
+
+        self._run_tp_on_model_cases(filtered_models)
+
+
+if __name__ == "__main__":
+    try:
+        mp.set_start_method("spawn")
+    except RuntimeError:
+        pass
+
+    unittest.main(warnings="ignore")

test/registered/lora/test_multi_lora_backend.py

@@ -0,0 +1,68 @@
+# Copyright 2023-2024 SGLang Team
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+import multiprocessing as mp
+import os
+import unittest
+
+from sglang.test.ci.ci_register import register_cuda_ci
+from sglang.test.lora_utils import (
+    ALL_OTHER_MULTI_LORA_MODELS,
+    CI_MULTI_LORA_MODELS,
+    run_lora_multiple_batch_on_model_cases,
+)
+from sglang.test.test_utils import CustomTestCase, is_in_ci
+
+register_cuda_ci(est_time=60, suite="stage-b-test-small-1-gpu")
+
+# All prompts are used at once in a batch.
+PROMPTS = [
+    "AI is a field of computer science focused on",
+    """
+    ### Instruction:
+    Tell me about llamas and alpacas
+    ### Response:
+    Llamas are large, long-necked animals with a woolly coat. They have two toes on each foot instead of three like other camelids.
+    ### Question:
+    What do you know about llamas?
+    ### Answer:
+    """,
+]
+
+
+class TestMultiLoRABackend(CustomTestCase):
+    def test_ci_lora_models(self):
+        run_lora_multiple_batch_on_model_cases(CI_MULTI_LORA_MODELS)
+
+    def test_all_lora_models(self):
+        if is_in_ci():
+            return
+
+        # Retain ONLY_RUN check here
+        filtered_models = []
+        for model_case in ALL_OTHER_MULTI_LORA_MODELS:
+            if "ONLY_RUN" in os.environ and os.environ["ONLY_RUN"] != model_case.base:
+                continue
+            filtered_models.append(model_case)
+
+        run_lora_multiple_batch_on_model_cases(filtered_models)
+
+
+if __name__ == "__main__":
+    try:
+        mp.set_start_method("spawn")
+    except RuntimeError:
+        pass
+
+    unittest.main(warnings="ignore")