chenchenghao/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[CI] Migrate LoRA tests to test/registered/lora/ (#15176)

Browse Source
This commit is contained in:
Alison Shao
2025-12-17 13:19:42 -08:00
committed by GitHub
parent 011d8d8970
commit 4128d4f5cb
18 changed files with 81 additions and 82 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/srt/lora/test_lora.py
-57
View File
@@ -1,57 +0,0 @@
# Copyright 2023-2024 SGLang Team
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
import multiprocessing as mp
import os
import sys
import unittest
from pathlib import Path
# Add test directory to path for lora_utils import
# TODO: can be removed after migration
sys.path.insert(0, str(Path(__file__).resolve().parent.parent.parent))
from lora_utils import (
ALL_OTHER_MULTI_LORA_MODELS,
CI_MULTI_LORA_MODELS,
run_lora_multiple_batch_on_model_cases,
)
from sglang.test.test_utils import CustomTestCase, is_in_ci
class TestLoRA(CustomTestCase):
def test_ci_lora_models(self):
run_lora_multiple_batch_on_model_cases(CI_MULTI_LORA_MODELS)
def test_all_lora_models(self):
if is_in_ci():
return
filtered_models = []
for model_case in ALL_OTHER_MULTI_LORA_MODELS:
if "ONLY_RUN" in os.environ and os.environ["ONLY_RUN"] != model_case.base:
continue
filtered_models.append(model_case)
run_lora_multiple_batch_on_model_cases(filtered_models)
if __name__ == "__main__":
try:
mp.set_start_method("spawn")
except RuntimeError:
pass
unittest.main(warnings="ignore")
test/srt/lora/test_lora_backend.py
-82
View File
@@ -1,82 +0,0 @@
# Copyright 2023-2024 SGLang Team
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
import multiprocessing as mp
import os
import sys
import unittest
from pathlib import Path
from typing import List
# Add test directory to path for lora_utils import
# TODO: can be removed after migration
sys.path.insert(0, str(Path(__file__).resolve().parent.parent.parent))
from lora_utils import (
ALL_OTHER_LORA_MODELS,
BACKENDS,
CI_LORA_MODELS,
DEFAULT_PROMPTS,
TORCH_DTYPES,
LoRAModelCase,
run_lora_test_one_by_one,
)
from sglang.test.test_utils import CustomTestCase, is_in_ci
class TestLoRABackend(CustomTestCase):
def _run_backend_on_model_cases(self, model_cases: List[LoRAModelCase]):
for model_case in model_cases:
# If skip_long_prompt is True, filter out prompts longer than 1000 characters
prompts = (
DEFAULT_PROMPTS
if not model_case.skip_long_prompt
else [p for p in DEFAULT_PROMPTS if len(p) < 1000]
)
for torch_dtype in TORCH_DTYPES:
for backend in BACKENDS:
run_lora_test_one_by_one(
prompts,
model_case,
torch_dtype,
max_new_tokens=32,
backend=backend,
)
def test_ci_lora_models(self):
self._run_backend_on_model_cases(CI_LORA_MODELS)
def test_all_lora_models(self):
if is_in_ci():
return
# Retain ONLY_RUN check here
filtered_models = []
for model_case in ALL_OTHER_LORA_MODELS:
if "ONLY_RUN" in os.environ and os.environ["ONLY_RUN"] != model_case.base:
continue
filtered_models.append(model_case)
self._run_backend_on_model_cases(filtered_models)
if __name__ == "__main__":
try:
mp.set_start_method("spawn")
except RuntimeError:
pass
unittest.main(warnings="ignore")
test/srt/lora/test_lora_eviction.py
-154
View File
@@ -1,154 +0,0 @@
# Copyright 2023-2024 SGLang Team
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
import contextlib
import multiprocessing as mp
import unittest
from typing import Dict, List, Tuple
import torch
from sglang.test.runners import SRTRunner
from sglang.test.test_utils import CustomTestCase
PROMPTS = [
"AI is a field of computer science focused on",
"""
### Instruction:
Compose a SQL query that uses the following table: users, and returns the user_id and name of all users whose name that does not have a duplicate in the table.
### Response:
SELECT user_id, name FROM users WHERE name LIKE 'A%';
""",
]
ADAPTERS = [
"faridlazuarda/valadapt-llama-3.1-8B-it-chinese", # target_modules = q, v
"philschmid/code-llama-3-1-8b-text-to-sql-lora", # target_modules = q, k, v, o, gate, up, down
]
BASE_MODEL = "meta-llama/Meta-Llama-3.1-8B-Instruct"
@contextlib.contextmanager
def dynamically_loaded_adapter(runner, lora_path: str, lora_name: str):
"""A context manager to load and automatically unload a LoRA adapter."""
try:
runner.load_lora_adapter(lora_name=lora_name, lora_path=lora_path)
yield
finally:
runner.unload_lora_adapter(lora_name=lora_name)
class TestLoRAEviction(CustomTestCase):
def test_lora_eviction_with_different_target_modules(self):
"""
Test LoRA eviction with different target modules.
This test runs inference against two LoRA adapters in different orders to force eviction behavior, and ensures
that the outputs of the same (adapter, prompt) pair are consistent across runs.
"""
output_history = {}
self._run_test(ADAPTERS, output_history, reverse=False)
self._run_test(ADAPTERS, output_history, reverse=True)
def test_lora_eviction_with_reused_lora_name(self):
"""
Test LoRA eviction with reused LoRA names.
This test runs inference against two LoRA adapters with the same name to ensure that the eviction behavior
works correctly when reusing LoRA names.
"""
output_history = {}
self._run_test(ADAPTERS, output_history, reuse_lora_name=True, repeat=1)
self._run_test(ADAPTERS, output_history, reuse_lora_name=False, repeat=1)
def _run_test(
self,
lora_paths: List[str],
output_history: Dict[Tuple[str, str], str],
reverse: bool = False,
repeat: int = 2,
reuse_lora_name: bool = False,
):
REUSED_LORA_NAME = "lora"
max_new_tokens = 256
torch_dtype = torch.float16
base_path = BASE_MODEL
assert len(lora_paths) >= 2
initial_lora_paths = lora_paths if not reuse_lora_name else None
# Initialize runners
with SRTRunner(
base_path,
torch_dtype=torch_dtype,
model_type="generation",
lora_paths=initial_lora_paths,
max_loras_per_batch=1,
enable_lora=True,
max_lora_rank=256,
# Need to list all lora modules, or "all" might include lora modules without assigning lora weights
# lora_target_modules=["all"],
lora_target_modules=[
"q_proj",
"k_proj",
"v_proj",
"o_proj",
"gate_proj",
"up_proj",
"down_proj",
],
) as srt_runner:
adapter_sequence = lora_paths if not reverse else lora_paths[::-1]
for i in range(repeat):
for j, lora_path in enumerate(adapter_sequence):
print(
f"\n========== Testing LoRA eviction with adapter '{lora_path}' (#{j + 1}/{len(adapter_sequence)}), reuse_lora_name: {reuse_lora_name}, reversed: {reverse}, repeat: {i + 1}/{repeat} ---"
)
lora_name = REUSED_LORA_NAME if reuse_lora_name else lora_path
context = (
dynamically_loaded_adapter(srt_runner, lora_path, lora_name)
if reuse_lora_name
else contextlib.nullcontext()
)
with context:
for prompt in PROMPTS:
print("\nprompt:\n", prompt)
srt_outputs = srt_runner.forward(
[prompt],
max_new_tokens=max_new_tokens,
lora_paths=[lora_name],
)
output = srt_outputs.output_strs[0].strip()
print("\noutput:\n", output)
prev_output = output_history.get((lora_path, prompt))
if prev_output is not None:
self.assertEqual(
prev_output,
output,
f"Output mismatch for adapter {lora_path} and prompt '{prompt}' on repeat {j + 1}, previous: '{prev_output}', current: '{output}'.",
)
else:
output_history[(lora_path, prompt)] = output
if __name__ == "__main__":
try:
mp.set_start_method("spawn")
except RuntimeError:
pass
unittest.main(warnings="ignore")
test/srt/lora/test_lora_hf_sgl_logprob_diff.py
-559
View File
@@ -1,559 +0,0 @@
# Copyright 2023-2024 SGLang Team
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""
Test to compare log probabilities between HuggingFace+LoRA and SGLang+LoRA.
This test:
1. Runs SGLang with LoRA and collects log probabilities
2. Runs HuggingFace with LoRA and collects log probabilities
3. Compares the differences (max and mean) between the two implementations
4. Uses unittest framework for easy integration with test suites
Usage:
python test_lora_hf_sgl_logprob_diff.py
or
python -m unittest test_lora_hf_sgl_logprob_diff
"""
import multiprocessing as mp
import os
import sys
import unittest
from typing import Any, Dict, List, Optional, Tuple
import numpy as np
import torch
# Add sglang to path if needed
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../python"))
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.runners import HFRunner, SRTRunner
register_cuda_ci(est_time=300, suite="nightly-1-gpu", nightly=True)
from sglang.test.test_utils import (
DEFAULT_PORT_FOR_SRT_TEST_RUNNER,
CustomTestCase,
is_in_ci,
)
# Test configuration constants
LORA_BACKEND = "triton"
DISABLE_CUDA_GRAPH = False
LORA_TARGET_MODULES = None
LOGPROB_THRESHOLD = 1e-01
# Default test prompts
DEFAULT_TEST_PROMPTS = [
"SGL is a",
"AI is a field of computer science focused on",
"Computer science is the study of",
"Write a short story.",
"What are the main components of a computer?",
]
# Formatting constants
DIVIDER_WIDTH = 80
SECTION_CHAR = "="
SUBSECTION_CHAR = "-"
def print_section_header(title: str):
"""Print a major section header."""
print("\n" + SECTION_CHAR * DIVIDER_WIDTH)
print(title)
print(SECTION_CHAR * DIVIDER_WIDTH)
def print_subsection_header(title: str):
"""Print a subsection header."""
print(f"\n{SUBSECTION_CHAR * 40}")
print(f"{title}")
print(SUBSECTION_CHAR * 40)
def print_config_info(title: str, config: Dict[str, Any]):
"""Print configuration information in a consistent format."""
print_section_header(title)
for key, value in config.items():
print(f" {key}: {value}")
def compare_logprobs_for_type(
sglang_logprobs: torch.Tensor, hf_logprobs: torch.Tensor, logprob_type: str
) -> Dict[str, Any]:
"""
Compare logprobs for a specific type (prefill or decode).
Args:
sglang_logprobs: SGLang log probabilities
hf_logprobs: HuggingFace log probabilities
logprob_type: Type of logprobs ("prefill" or "decode")
Returns:
Dictionary containing comparison statistics
"""
diff = torch.abs(sglang_logprobs - hf_logprobs)
max_diff = torch.max(diff).item()
mean_diff = torch.mean(diff).item()
shape = list(sglang_logprobs.shape)
matches_threshold = max_diff < LOGPROB_THRESHOLD
return {
"max_diff": max_diff,
"mean_diff": mean_diff,
"shape": shape,
"matches_threshold": matches_threshold,
"type": logprob_type,
}
def print_logprob_comparison(comparison: Dict[str, Any]):
"""Print logprob comparison results in a consistent format."""
logprob_type = comparison["type"].capitalize()
print(f"\n{logprob_type} logprobs:")
print(f" Shape: {comparison['shape']}")
print(f" Max difference: {comparison['max_diff']:.6e}")
print(f" Mean difference: {comparison['mean_diff']:.6e}")
status = "PASS" if comparison["matches_threshold"] else "FAIL"
print(f" Status: {status} (threshold: {LOGPROB_THRESHOLD:.0e})")
def compare_output_strings(
sglang_output: str, hf_output: str, max_display_len: int = 200
) -> Dict[str, Any]:
"""
Compare output strings between SGLang and HuggingFace.
Args:
sglang_output: SGLang generated text
hf_output: HuggingFace generated text
max_display_len: Maximum length for display
Returns:
Dictionary containing comparison results
"""
outputs_match = sglang_output.strip() == hf_output.strip()
# Truncate for display if needed
sglang_display = (
sglang_output[:max_display_len]
if len(sglang_output) > max_display_len
else sglang_output
)
hf_display = (
hf_output[:max_display_len] if len(hf_output) > max_display_len else hf_output
)
return {
"match": outputs_match,
"sglang_output": sglang_output,
"hf_output": hf_output,
"sglang_display": sglang_display,
"hf_display": hf_display,
}
def print_output_comparison(comparison: Dict[str, Any]):
"""Print output string comparison in a consistent format."""
print(f"\nOutput strings:")
status = "MATCH" if comparison["match"] else "DIFFER"
print(f" Status: {status}")
print(f" SGLang: {comparison['sglang_display']}")
print(f" HuggingFace: {comparison['hf_display']}")
def prepare_lora_paths_per_prompt(
lora_paths: List[str], num_prompts: int
) -> List[Optional[str]]:
"""
Prepare LoRA paths for each prompt by cycling through available LoRAs.
Args:
lora_paths: List of available LoRA adapter paths
num_prompts: Number of prompts to generate LoRA paths for
Returns:
List of LoRA paths (one per prompt), or None values if no LoRAs
"""
if not lora_paths:
return [None] * num_prompts
return [lora_paths[i % len(lora_paths)] for i in range(num_prompts)]
def run_sglang_with_lora(
model_path: str,
lora_paths: List[str],
prompts: List[str],
max_new_tokens: int,
torch_dtype: torch.dtype,
lora_backend: str,
port: int,
disable_cuda_graph: bool,
lora_target_modules: Optional[List[str]],
tp_size: int,
) -> Dict[str, Any]:
"""Run SGLang with LoRA and return log probabilities."""
config = {
"Model": model_path,
"LoRA paths": lora_paths,
"LoRA backend": lora_backend,
"Disable CUDA graph": disable_cuda_graph,
"Port": port,
"Number of prompts": len(prompts),
"Tensor parallel size": tp_size,
}
print_config_info("Running SGLang with LoRA", config)
lora_paths_per_prompt = prepare_lora_paths_per_prompt(lora_paths, len(prompts))
with SRTRunner(
model_path,
torch_dtype=torch_dtype,
model_type="generation",
tp_size=tp_size,
lora_paths=lora_paths,
max_loras_per_batch=len(lora_paths) if lora_paths else 1,
lora_backend=lora_backend,
disable_cuda_graph=disable_cuda_graph,
disable_radix_cache=True,
port=port,
mem_fraction_static=0.88,
lora_target_modules=lora_target_modules,
) as srt_runner:
srt_outputs = srt_runner.forward(
prompts,
max_new_tokens=max_new_tokens,
lora_paths=lora_paths_per_prompt,
)
return {
"top_input_logprobs": srt_outputs.top_input_logprobs,
"top_output_logprobs": srt_outputs.top_output_logprobs,
"output_strs": srt_outputs.output_strs,
"lora_paths": lora_paths_per_prompt,
}
def run_hf_with_lora(
model_path: str,
lora_paths: List[str],
prompts: List[str],
max_new_tokens: int,
torch_dtype: torch.dtype,
) -> Dict[str, Any]:
"""Run HuggingFace with LoRA and return log probabilities."""
config = {
"Model": model_path,
"LoRA paths": lora_paths,
"Number of prompts": len(prompts),
}
print_config_info("Running HuggingFace with LoRA", config)
lora_paths_per_prompt = prepare_lora_paths_per_prompt(lora_paths, len(prompts))
with HFRunner(
model_path,
torch_dtype=torch_dtype,
model_type="generation",
patch_model_do_sample_false=True,
) as hf_runner:
hf_outputs = hf_runner.forward(
prompts,
max_new_tokens=max_new_tokens,
lora_paths=lora_paths_per_prompt,
)
return {
"top_input_logprobs": hf_outputs.top_input_logprobs,
"top_output_logprobs": hf_outputs.top_output_logprobs,
"output_strs": hf_outputs.output_strs,
"lora_paths": lora_paths_per_prompt,
}
def compare_single_prompt(
prompt_idx: int,
sglang_data: Dict[str, Any],
hf_data: Dict[str, Any],
) -> Dict[str, Any]:
"""
Compare logprobs and outputs for a single prompt.
Args:
prompt_idx: Index of the prompt being compared
sglang_data: SGLang results data
hf_data: HuggingFace results data
Returns:
Dictionary containing all comparison results
"""
print_subsection_header(f"Prompt {prompt_idx + 1}")
print(f"LoRA adapter: {sglang_data['lora_paths'][prompt_idx]}")
result = {
"prompt_idx": prompt_idx,
"lora_path": sglang_data["lora_paths"][prompt_idx],
}
# Compare prefill (input) logprobs
sglang_prefill = torch.tensor(sglang_data["top_input_logprobs"][prompt_idx])
hf_prefill = torch.tensor(hf_data["top_input_logprobs"][prompt_idx])
prefill_comparison = compare_logprobs_for_type(
sglang_prefill, hf_prefill, "prefill"
)
print_logprob_comparison(prefill_comparison)
# Store prefill results
result["prefill_max_diff"] = prefill_comparison["max_diff"]
result["prefill_mean_diff"] = prefill_comparison["mean_diff"]
result["prefill_shape"] = prefill_comparison["shape"]
result["prefill_logprob_match"] = prefill_comparison["matches_threshold"]
# Compare decode (output) logprobs
sglang_decode = torch.tensor(sglang_data["top_output_logprobs"][prompt_idx])
hf_decode = torch.tensor(hf_data["top_output_logprobs"][prompt_idx])
decode_comparison = compare_logprobs_for_type(sglang_decode, hf_decode, "decode")
print_logprob_comparison(decode_comparison)
# Store decode results
result["decode_max_diff"] = decode_comparison["max_diff"]
result["decode_mean_diff"] = decode_comparison["mean_diff"]
result["decode_shape"] = decode_comparison["shape"]
result["decode_logprob_match"] = decode_comparison["matches_threshold"]
# Overall logprob match
result["overall_logprob_match"] = (
prefill_comparison["matches_threshold"]
and decode_comparison["matches_threshold"]
)
# Compare output strings
sglang_output = sglang_data["output_strs"][prompt_idx]
hf_output = hf_data["output_strs"][prompt_idx]
output_comparison = compare_output_strings(sglang_output, hf_output)
print_output_comparison(output_comparison)
# Store output results
result["outputs_match"] = output_comparison["match"]
result["sglang_output"] = output_comparison["sglang_output"]
result["hf_output"] = output_comparison["hf_output"]
return result
def print_overall_statistics(results: List[Dict[str, Any]]):
"""Print overall statistics across all prompts."""
print_section_header("Overall Statistics")
# Gather statistics
prefill_max_diffs = [r["prefill_max_diff"] for r in results]
prefill_mean_diffs = [r["prefill_mean_diff"] for r in results]
decode_max_diffs = [r["decode_max_diff"] for r in results]
decode_mean_diffs = [r["decode_mean_diff"] for r in results]
# Print logprob statistics
print("\nLogprob Differences:")
print(f" Prefill:")
print(f" Max of max: {max(prefill_max_diffs):.6e}")
print(f" Mean of max: {np.mean(prefill_max_diffs):.6e}")
print(f" Mean of mean: {np.mean(prefill_mean_diffs):.6e}")
print(f" Decode:")
print(f" Max of max: {max(decode_max_diffs):.6e}")
print(f" Mean of max: {np.mean(decode_max_diffs):.6e}")
print(f" Mean of mean: {np.mean(decode_mean_diffs):.6e}")
# Print match statistics
num_prompts = len(results)
logprob_match_count = sum(r["overall_logprob_match"] for r in results)
prefill_match_count = sum(r["prefill_logprob_match"] for r in results)
decode_match_count = sum(r["decode_logprob_match"] for r in results)
outputs_match_count = sum(r["outputs_match"] for r in results)
print(f"\nLogprob Statistics (threshold: {LOGPROB_THRESHOLD:.0e}):")
overall_status = "PASSED" if logprob_match_count == num_prompts else "FAILED"
print(f" Overall logprob: {logprob_match_count}/{num_prompts} {overall_status}")
print(f" Prefill logprob: {prefill_match_count}/{num_prompts}")
print(f" Decode logprob: {decode_match_count}/{num_prompts}")
print(f"\nString Statistics:")
print(f" Output strings: {outputs_match_count}/{num_prompts}")
# Return overall stats for saving
return {
"logprob_differences": {
"prefill": {
"max_of_max_diffs": max(prefill_max_diffs),
"mean_of_max_diffs": float(np.mean(prefill_max_diffs)),
"mean_of_mean_diffs": float(np.mean(prefill_mean_diffs)),
},
"decode": {
"max_of_max_diffs": max(decode_max_diffs),
"mean_of_max_diffs": float(np.mean(decode_max_diffs)),
"mean_of_mean_diffs": float(np.mean(decode_mean_diffs)),
},
},
"match_statistics": {
"overall_logprob_match_rate": logprob_match_count / num_prompts,
"prefill_logprob_match_rate": prefill_match_count / num_prompts,
"decode_logprob_match_rate": decode_match_count / num_prompts,
"outputs_match_rate": outputs_match_count / num_prompts,
},
}
def compare_logprobs(
sglang_logprobs: Dict[str, Any], hf_logprobs: Dict[str, Any]
) -> Tuple[List[Dict[str, Any]], Dict[str, Any]]:
"""Compare log probabilities and compute statistics."""
print_section_header("Comparing Log Probabilities")
results = []
num_prompts = len(sglang_logprobs["top_input_logprobs"])
for i in range(num_prompts):
result = compare_single_prompt(i, sglang_logprobs, hf_logprobs)
results.append(result)
overall_stats = print_overall_statistics(results)
return results, overall_stats
class TestLoRAHFSGLLogprobDifference(CustomTestCase):
"""
Test case to compare log probabilities between HuggingFace+LoRA and SGLang+LoRA.
"""
def _run_comparison_test(
self,
model_path: str,
lora_paths: List[str],
prompts: List[str],
max_new_tokens: int = 32,
torch_dtype: torch.dtype = torch.float16,
lora_backend: str = LORA_BACKEND,
port: int = DEFAULT_PORT_FOR_SRT_TEST_RUNNER,
disable_cuda_graph: bool = DISABLE_CUDA_GRAPH,
lora_target_modules: Optional[List[str]] = LORA_TARGET_MODULES,
tp_size: int = 1,
):
"""
Run comparison test between SGLang and HuggingFace with LoRA.
"""
print_section_header(f"Testing {model_path} with LoRA adapters")
# Step 1: Run SGLang with LoRA
sglang_logprobs = run_sglang_with_lora(
model_path=model_path,
lora_paths=lora_paths,
prompts=prompts,
max_new_tokens=max_new_tokens,
torch_dtype=torch_dtype,
lora_backend=lora_backend,
port=port,
disable_cuda_graph=disable_cuda_graph,
lora_target_modules=lora_target_modules,
tp_size=tp_size,
)
# Clear GPU memory
print("\nClearing GPU memory...")
torch.cuda.empty_cache()
# Step 2: Run HuggingFace with LoRA
hf_logprobs = run_hf_with_lora(
model_path=model_path,
lora_paths=lora_paths,
prompts=prompts,
max_new_tokens=max_new_tokens,
torch_dtype=torch_dtype,
)
# Step 3: Compare log probabilities
results, overall_stats = compare_logprobs(sglang_logprobs, hf_logprobs)
# Assert that all prompts pass the threshold
for result in results:
self.assertTrue(
result["prefill_logprob_match"],
f"Prefill logprob mismatch for prompt {result['prompt_idx']} "
f"(max_diff={result['prefill_max_diff']:.6e}, threshold={LOGPROB_THRESHOLD:.0e})",
)
self.assertTrue(
result["decode_logprob_match"],
f"Decode logprob mismatch for prompt {result['prompt_idx']} "
f"(max_diff={result['decode_max_diff']:.6e}, threshold={LOGPROB_THRESHOLD:.0e})",
)
print_section_header("Test completed successfully!")
return results, overall_stats
def test_lora_logprob_comparison_basic(self):
"""
Basic test comparing HF and SGLang LoRA logprobs with small model.
"""
# Use a smaller model and shorter prompts for CI
if is_in_ci():
self.skipTest("Skipping in CI environment - requires large models")
model_path = "meta-llama/Llama-2-7b-hf"
lora_paths = ["yushengsu/sglang_lora_logprob_diff_without_tuning"]
prompts = DEFAULT_TEST_PROMPTS[:2] # Use fewer prompts for faster testing
self._run_comparison_test(
model_path=model_path,
lora_paths=lora_paths,
prompts=prompts,
max_new_tokens=32,
)
def test_lora_logprob_comparison_full(self):
"""
Full test comparing HF and SGLang LoRA logprobs with all prompts.
"""
if is_in_ci():
self.skipTest("Skipping in CI environment - requires large models")
model_path = "meta-llama/Llama-2-7b-hf"
lora_paths = ["yushengsu/sglang_lora_logprob_diff_without_tuning"]
prompts = DEFAULT_TEST_PROMPTS
self._run_comparison_test(
model_path=model_path,
lora_paths=lora_paths,
prompts=prompts,
max_new_tokens=32,
)
if __name__ == "__main__":
try:
mp.set_start_method("spawn")
except RuntimeError:
pass
try:
unittest.main(warnings="ignore", verbosity=2)
finally:
# Final cleanup
if torch.cuda.is_available():
torch.cuda.empty_cache()
torch.cuda.synchronize()
test/srt/lora/test_lora_tp.py
-83
View File
@@ -1,83 +0,0 @@
# Copyright 2023-2024 SGLang Team
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
import multiprocessing as mp
import os
import sys
import unittest
from pathlib import Path
from typing import List
# Add test directory to path for lora_utils import
# TODO: can be removed after migration
sys.path.insert(0, str(Path(__file__).resolve().parent.parent.parent))
from lora_utils import (
ALL_OTHER_LORA_MODELS,
CI_LORA_MODELS,
DEFAULT_PROMPTS,
TORCH_DTYPES,
LoRAModelCase,
run_lora_test_one_by_one,
)
from sglang.test.test_utils import CustomTestCase, is_in_ci
class TestLoRATP(CustomTestCase):
def _run_tp_on_model_cases(self, model_cases: List[LoRAModelCase]):
tp_list = [2] # Define TP sizes to iterate over
for model_case in model_cases:
# If skip_long_prompt is True, filter out prompts longer than 1000 characters
prompts = (
DEFAULT_PROMPTS
if not model_case.skip_long_prompt
else [p for p in DEFAULT_PROMPTS if len(p) < 1000]
)
for tp_size in tp_list:
model_case.tp_size = tp_size
for torch_dtype in TORCH_DTYPES:
run_lora_test_one_by_one(
prompts,
model_case,
torch_dtype,
max_new_tokens=32,
test_tag=f"tp={tp_size}",
)
def test_ci_lora_models(self):
self._run_tp_on_model_cases(CI_LORA_MODELS)
def test_all_lora_models(self):
if is_in_ci():
return
# Retain ONLY_RUN check here
filtered_models = []
for model_case in ALL_OTHER_LORA_MODELS:
if "ONLY_RUN" in os.environ and os.environ["ONLY_RUN"] != model_case.base:
continue
filtered_models.append(model_case)
self._run_tp_on_model_cases(filtered_models)
if __name__ == "__main__":
try:
mp.set_start_method("spawn")
except RuntimeError:
pass
unittest.main(warnings="ignore")
test/srt/lora/test_lora_update.py
-1601
View File
File diff suppressed because it is too large Load Diff
test/srt/lora/test_multi_lora_backend.py
-72
View File
@@ -1,72 +0,0 @@
# Copyright 2023-2024 SGLang Team
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
import multiprocessing as mp
import os
import sys
import unittest
from pathlib import Path
# Add test directory to path for lora_utils import
# TODO: can be removed after migration
sys.path.insert(0, str(Path(__file__).resolve().parent.parent.parent))
from lora_utils import (
ALL_OTHER_MULTI_LORA_MODELS,
CI_MULTI_LORA_MODELS,
run_lora_multiple_batch_on_model_cases,
)
from sglang.test.test_utils import CustomTestCase, is_in_ci
# All prompts are used at once in a batch.
PROMPTS = [
"AI is a field of computer science focused on",
"""
### Instruction:
Tell me about llamas and alpacas
### Response:
Llamas are large, long-necked animals with a woolly coat. They have two toes on each foot instead of three like other camelids.
### Question:
What do you know about llamas?
### Answer:
""",
]
class TestMultiLoRABackend(CustomTestCase):
def test_ci_lora_models(self):
run_lora_multiple_batch_on_model_cases(CI_MULTI_LORA_MODELS)
def test_all_lora_models(self):
if is_in_ci():
return
# Retain ONLY_RUN check here
filtered_models = []
for model_case in ALL_OTHER_MULTI_LORA_MODELS:
if "ONLY_RUN" in os.environ and os.environ["ONLY_RUN"] != model_case.base:
continue
filtered_models.append(model_case)
run_lora_multiple_batch_on_model_cases(filtered_models)
if __name__ == "__main__":
try:
mp.set_start_method("spawn")
except RuntimeError:
pass
unittest.main(warnings="ignore")
test/srt/run_suite.py
+2 -12
View File
@@ -13,11 +13,6 @@ suites = {
TestFile("layers/attention/mamba/test_causal_conv1d.py", 25),
TestFile("layers/attention/mamba/test_mamba_ssm.py", 7),
TestFile("layers/attention/mamba/test_mamba_ssm_ssd.py", 13),
TestFile("lora/test_lora.py", 82),
TestFile("lora/test_lora_eviction.py", 224),
TestFile("lora/test_lora_update.py", 451),
TestFile("lora/test_lora_backend.py", 200),
TestFile("lora/test_multi_lora_backend.py", 60),
TestFile("models/test_compressed_tensors_models.py", 42),
TestFile("models/test_cross_encoder_models.py", 100),
TestFile("models/test_embedding_models.py", 73),
@@ -133,7 +128,6 @@ suites = {
TestFile("hicache/test_hicache_storage_file_backend.py", 200),
TestFile("hicache/test_hicache_storage_mooncake_backend.py", 300),
TestFile("layers/attention/mamba/test_mamba2_mixer.py", 50),
TestFile("lora/test_lora_tp.py", 116),
TestFile("models/test_glm4_moe_models.py", 100),
TestFile("models/test_kimi_linear_models.py", 90),
TestFile("rl/test_update_weights_from_distributed.py", 103),
@@ -201,10 +195,8 @@ suites = {
TestFile("test_quantization.py", 185),
TestFile("test_gguf.py", 96),
],
# Nightly test suites have been moved to test/run_suite_nightly.py
"__not_in_ci__": [
TestFile("test_release_memory_occupation.py", 200), # Temporarily disabled
TestFile("lora/test_lora_hf_sgl_logprob_diff.py"), # Nightly test
TestFile("models/test_dummy_grok_models.py"),
TestFile(
"rl/test_update_weights_from_disk.py"
@@ -228,12 +220,10 @@ suite_amd = {
# TestFile("hicache/test_hicache.py", 116), # Disabled temporarily, see https://github.com/sgl-project/sglang/issues/12575
# TestFile("hicache/test_hicache_mla.py", 127), # Disabled temporarily, # Temporarily disabled, see https://github.com/sgl-project/sglang/issues/12574
# TestFile("hicache/test_hicache_storage.py", 127), # Disabled temporarily, see https://github.com/sgl-project/sglang/issues/12575
TestFile("lora/test_lora.py", 665),
# LoRA tests moved to test/registered/lora/ - AMD entries need to be re-added there
# TestFile("lora/test_lora_backend.py", 99), # Disabled temporarily, see https://github.com/sgl-project/sglang/issues/13107
# TestFile("lora/test_lora_cuda_graph.py", 250), # Disabled temporarily, see https://github.com/sgl-project/sglang/issues/13107
TestFile("lora/test_lora_eviction.py", 240),
# TestFile("lora/test_lora_qwen3.py", 97), # Disabled temporarily, see https://github.com/sgl-project/sglang/issues/13107
TestFile("lora/test_multi_lora_backend.py", 60),
TestFile("models/test_compressed_tensors_models.py", 42),
TestFile("models/test_qwen_models.py", 82),
TestFile("models/test_reward_models.py", 132),
@@ -308,7 +298,7 @@ suite_amd = {
TestFile("test_mla.py", 242),
],
"per-commit-2-gpu-amd": [
# TestFile("lora/test_lora_tp.py", 116), # Disabled temporarily, see https://github.com/sgl-project/sglang/issues/13107
# TestFile("lora/test_lora_tp.py", 116), # Disabled temporarily, see https://github.com/sgl-project/sglang/issues/13107. Moved to test/registered/lora/
TestFile("rl/test_update_weights_from_distributed.py", 103),
TestFile("test_data_parallelism.py", 73),
TestFile("test_load_weights_from_remote_instance.py", 72),