chenchenghao/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[TestFix] change LoRA tests to use NVIDIA adapter instead of Nutanix (#19642)

Browse Source 
Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
This commit is contained in:
Glen Liu
2026-03-02 15:55:41 -05:00
committed by GitHub
parent 51ee17ce44
commit cc860a2198
3 changed files with 60 additions and 81 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
python/sglang/test/lora_utils.py
View File

@@ -68,7 +68,7 @@ ALL_OTHER_LORA_MODELS = [
base="meta-llama/Llama-3.1-8B-Instruct",
adaptors=[
LoRAAdaptor(
name="Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16",
name="nvidia/llama-3.1-nemoguard-8b-topic-control",
prefill_tolerance=1e-1,
),
],
@@ -109,7 +109,7 @@ ALL_OTHER_MULTI_LORA_MODELS = [
prefill_tolerance=1e-1,
),
LoRAAdaptor(
name="Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16",
name="nvidia/llama-3.1-nemoguard-8b-topic-control",
prefill_tolerance=1e-1,
),
],

127
test/registered/lora/test_lora_update.py
View File

@@ -37,7 +37,6 @@ from sglang.test.test_utils import (
register_cuda_ci(
est_time=487,
suite="stage-b-test-large-1-gpu",
disabled="Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16 removed from HuggingFace",
)
PROMPTS = [
@@ -98,13 +97,13 @@ BASIC_TESTS = [
max_loras_per_batch=3,
all_adapters=[
"philschmid/code-llama-3-1-8b-text-to-sql-lora",
"Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16",
"nvidia/llama-3.1-nemoguard-8b-topic-control",
"pbevan11/llama-3.1-8b-ocr-correction",
],
initial_adapters=[
# Testing 3 supported lora-path formats.
"philschmid/code-llama-3-1-8b-text-to-sql-lora",
"Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16=Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16",
"nvidia/llama-3.1-nemoguard-8b-topic-control=nvidia/llama-3.1-nemoguard-8b-topic-control",
{
"lora_name": "pbevan11/llama-3.1-8b-ocr-correction",
"lora_path": "pbevan11/llama-3.1-8b-ocr-correction",
@@ -130,14 +129,14 @@ BASIC_TESTS = [
data=create_batch_data(
[
"philschmid/code-llama-3-1-8b-text-to-sql-lora",
"Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16",
"nvidia/llama-3.1-nemoguard-8b-topic-control",
"pbevan11/llama-3.1-8b-ocr-correction",
]
),
),
Operation(
type=OperationType.UNLOAD,
data="Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16",
data="nvidia/llama-3.1-nemoguard-8b-topic-control",
),
Operation(
type=OperationType.UNLOAD,
@@ -149,9 +148,7 @@ BASIC_TESTS = [
),
Operation(
type=OperationType.FORWARD,
data=create_batch_data(
"Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16"
),
data=create_batch_data("nvidia/llama-3.1-nemoguard-8b-topic-control"),
),
Operation(
type=OperationType.FORWARD,
@@ -159,7 +156,7 @@ BASIC_TESTS = [
),
Operation(
type=OperationType.LOAD,
data="Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16",
data="nvidia/llama-3.1-nemoguard-8b-topic-control",
expected_error="already loaded",
),
Operation(
@@ -172,7 +169,7 @@ BASIC_TESTS = [
data=create_batch_data(
[
"philschmid/code-llama-3-1-8b-text-to-sql-lora",
"Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16",
"nvidia/llama-3.1-nemoguard-8b-topic-control",
"pbevan11/llama-3.1-8b-ocr-correction",
]
),
@@ -189,14 +186,14 @@ BASIC_TESTS = [
type=OperationType.FORWARD,
data=create_batch_data(
[
"Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16",
"nvidia/llama-3.1-nemoguard-8b-topic-control",
"pbevan11/llama-3.1-8b-ocr-correction",
]
),
),
Operation(
type=OperationType.UNLOAD,
data="Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16",
data="nvidia/llama-3.1-nemoguard-8b-topic-control",
),
Operation(
type=OperationType.UNLOAD,
@@ -204,9 +201,7 @@ BASIC_TESTS = [
),
Operation(
type=OperationType.FORWARD,
data=create_batch_data(
"Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16"
),
data=create_batch_data("nvidia/llama-3.1-nemoguard-8b-topic-control"),
),
Operation(
type=OperationType.FORWARD,
@@ -229,7 +224,7 @@ BASIC_TESTS = [
max_loras_per_batch=4,
all_adapters=[
"philschmid/code-llama-3-1-8b-text-to-sql-lora",
"Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16",
"nvidia/llama-3.1-nemoguard-8b-topic-control",
"pbevan11/llama-3.1-8b-ocr-correction",
],
op_sequence=[
@@ -239,7 +234,7 @@ BASIC_TESTS = [
),
Operation(
type=OperationType.LOAD,
data="Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16",
data="nvidia/llama-3.1-nemoguard-8b-topic-control",
),
Operation(
type=OperationType.LOAD,
@@ -263,7 +258,7 @@ BASIC_TESTS = [
data=create_batch_data(
[
"philschmid/code-llama-3-1-8b-text-to-sql-lora",
"Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16",
"nvidia/llama-3.1-nemoguard-8b-topic-control",
"pbevan11/llama-3.1-8b-ocr-correction",
None,
]
@@ -282,7 +277,7 @@ BASIC_TESTS = [
data=create_batch_data(
[
None,
"Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16",
"nvidia/llama-3.1-nemoguard-8b-topic-control",
"pbevan11/llama-3.1-8b-ocr-correction",
None,
]
@@ -290,7 +285,7 @@ BASIC_TESTS = [
),
Operation(
type=OperationType.UNLOAD,
data="Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16",
data="nvidia/llama-3.1-nemoguard-8b-topic-control",
),
Operation(
type=OperationType.UNLOAD,
@@ -298,9 +293,7 @@ BASIC_TESTS = [
),
Operation(
type=OperationType.FORWARD,
data=create_batch_data(
"Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16"
),
data=create_batch_data("nvidia/llama-3.1-nemoguard-8b-topic-control"),
),
Operation(
type=OperationType.FORWARD,
@@ -317,7 +310,7 @@ BASIC_TESTS = [
),
Operation(
type=OperationType.LOAD,
data="Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16",
data="nvidia/llama-3.1-nemoguard-8b-topic-control",
expected_error="already loaded",
),
Operation(
@@ -330,7 +323,7 @@ BASIC_TESTS = [
data=create_batch_data(
[
"philschmid/code-llama-3-1-8b-text-to-sql-lora",
"Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16",
"nvidia/llama-3.1-nemoguard-8b-topic-control",
"pbevan11/llama-3.1-8b-ocr-correction",
None,
]
@@ -347,7 +340,7 @@ TARGET_MODULE_TESTS = [
lora_target_modules=["all"],
max_lora_rank=64,
all_adapters=[
"Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", # target_modules = q, k, v, o, gate, up, down
"nvidia/llama-3.1-nemoguard-8b-topic-control", # target_modules = q, k, v, o, gate, up, down
"algoprog/fact-generation-llama-3.1-8b-instruct-lora", # target_modules = q, k, v, o, gate
],
initial_adapters=["algoprog/fact-generation-llama-3.1-8b-instruct-lora"],
@@ -360,21 +353,19 @@ TARGET_MODULE_TESTS = [
),
Operation(
type=OperationType.FORWARD,
data=create_batch_data(
"Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16"
),
data=create_batch_data("nvidia/llama-3.1-nemoguard-8b-topic-control"),
expected_error="never been loaded",
),
Operation(
type=OperationType.LOAD,
data="Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16",
data="nvidia/llama-3.1-nemoguard-8b-topic-control",
),
Operation(
type=OperationType.FORWARD,
data=create_batch_data(
[
"algoprog/fact-generation-llama-3.1-8b-instruct-lora",
"Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16",
"nvidia/llama-3.1-nemoguard-8b-topic-control",
None,
]
),
@@ -387,16 +378,14 @@ TARGET_MODULE_TESTS = [
max_loras_per_batch=3,
max_lora_rank=64,
all_adapters=[
"Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", # target_modules = q, k, v, o, gate, up, down
"nvidia/llama-3.1-nemoguard-8b-topic-control", # target_modules = q, k, v, o, gate, up, down
"algoprog/fact-generation-llama-3.1-8b-instruct-lora", # target_modules = q, k, v, o, gate
],
initial_adapters=["Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16"],
initial_adapters=["nvidia/llama-3.1-nemoguard-8b-topic-control"],
op_sequence=[
Operation(
type=OperationType.FORWARD,
data=create_batch_data(
"Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16"
),
data=create_batch_data("nvidia/llama-3.1-nemoguard-8b-topic-control"),
),
Operation(
type=OperationType.FORWARD,
@@ -414,7 +403,7 @@ TARGET_MODULE_TESTS = [
data=create_batch_data(
[
"algoprog/fact-generation-llama-3.1-8b-instruct-lora",
"Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16",
"nvidia/llama-3.1-nemoguard-8b-topic-control",
None,
]
),
@@ -427,7 +416,7 @@ TARGET_MODULE_TESTS = [
max_loras_per_batch=3,
max_lora_rank=64,
all_adapters=[
"Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", # target_modules = q, k, v, o, gate, up, down
"nvidia/llama-3.1-nemoguard-8b-topic-control", # target_modules = q, k, v, o, gate, up, down
"algoprog/fact-generation-llama-3.1-8b-instruct-lora", # target_modules = q, k, v, o, gate
],
initial_adapters=["algoprog/fact-generation-llama-3.1-8b-instruct-lora"],
@@ -440,14 +429,12 @@ TARGET_MODULE_TESTS = [
),
Operation(
type=OperationType.FORWARD,
data=create_batch_data(
"Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16"
),
data=create_batch_data("nvidia/llama-3.1-nemoguard-8b-topic-control"),
expected_error="never been loaded",
),
Operation(
type=OperationType.LOAD,
data="Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16",
data="nvidia/llama-3.1-nemoguard-8b-topic-control",
expected_error="incompatible",
),
Operation(
@@ -469,17 +456,15 @@ MAX_LORA_RANK_TESTS = [
max_loras_per_batch=3,
max_lora_rank=32,
all_adapters=[
"Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", # r = 4
"nvidia/llama-3.1-nemoguard-8b-topic-control", # r = 4
"pbevan11/llama-3.1-8b-ocr-correction", # r = 32
"philschmid/code-llama-3-1-8b-text-to-sql-lora", # r = 256
],
initial_adapters=["Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16"],
initial_adapters=["nvidia/llama-3.1-nemoguard-8b-topic-control"],
op_sequence=[
Operation(
type=OperationType.FORWARD,
data=create_batch_data(
"Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16"
),
data=create_batch_data("nvidia/llama-3.1-nemoguard-8b-topic-control"),
),
Operation(
type=OperationType.FORWARD,
@@ -500,7 +485,7 @@ MAX_LORA_RANK_TESTS = [
data=create_batch_data(
[
"pbevan11/llama-3.1-8b-ocr-correction",
"Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16",
"nvidia/llama-3.1-nemoguard-8b-topic-control",
None,
]
),
@@ -522,7 +507,7 @@ MAX_LORA_RANK_TESTS = [
data=create_batch_data(
[
"pbevan11/llama-3.1-8b-ocr-correction",
"Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16",
"nvidia/llama-3.1-nemoguard-8b-topic-control",
None,
]
),
@@ -534,7 +519,7 @@ MAX_LORA_RANK_TESTS = [
base="meta-llama/Llama-3.1-8B-Instruct",
max_loras_per_batch=3,
all_adapters=[
"Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", # r = 4
"nvidia/llama-3.1-nemoguard-8b-topic-control", # r = 4
"pbevan11/llama-3.1-8b-ocr-correction", # r = 32
"philschmid/code-llama-3-1-8b-text-to-sql-lora", # r = 256
],
@@ -556,19 +541,19 @@ MAX_LORA_RANK_TESTS = [
),
Operation(
type=OperationType.LOAD,
data="Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16",
data="nvidia/llama-3.1-nemoguard-8b-topic-control",
),
Operation(
type=OperationType.FORWARD,
data=create_batch_data(
"Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16",
"nvidia/llama-3.1-nemoguard-8b-topic-control",
),
),
Operation(
type=OperationType.FORWARD,
data=create_batch_data(
[
"Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16",
"nvidia/llama-3.1-nemoguard-8b-topic-control",
"pbevan11/llama-3.1-8b-ocr-correction",
None,
]
@@ -585,14 +570,14 @@ MAX_LOADED_LORAS_TESTS = [
max_loaded_loras=2,
all_adapters=[
"philschmid/code-llama-3-1-8b-text-to-sql-lora",
"Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16",
"nvidia/llama-3.1-nemoguard-8b-topic-control",
"pbevan11/llama-3.1-8b-ocr-correction",
],
initial_adapters=["philschmid/code-llama-3-1-8b-text-to-sql-lora"],
op_sequence=[
Operation(
type=OperationType.LOAD,
data="Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16",
data="nvidia/llama-3.1-nemoguard-8b-topic-control",
),
Operation(
type=OperationType.LOAD,
@@ -606,7 +591,7 @@ MAX_LOADED_LORAS_TESTS = [
type=OperationType.FORWARD,
data=create_batch_data(
[
"Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16",
"nvidia/llama-3.1-nemoguard-8b-topic-control",
"philschmid/code-llama-3-1-8b-text-to-sql-lora",
]
),
@@ -620,7 +605,7 @@ MAX_LOADED_LORAS_TESTS = [
type=OperationType.FORWARD,
data=create_batch_data(
[
"Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16",
"nvidia/llama-3.1-nemoguard-8b-topic-control",
]
),
),
@@ -628,13 +613,13 @@ MAX_LOADED_LORAS_TESTS = [
type=OperationType.LOAD,
data="philschmid/code-llama-3-1-8b-text-to-sql-lora",
),
# Implicitly load "pbevan11/llama-3.1-8b-ocr-correction" and make sure that "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16"
# Implicitly load "pbevan11/llama-3.1-8b-ocr-correction" and make sure that "nvidia/llama-3.1-nemoguard-8b-topic-control"
# isn't implicitly unloaded even though it is LRU because it is needed for this forward pass
Operation(
type=OperationType.FORWARD,
data=create_batch_data(
[
"Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16",
"nvidia/llama-3.1-nemoguard-8b-topic-control",
"pbevan11/llama-3.1-8b-ocr-correction",
]
),
@@ -644,7 +629,7 @@ MAX_LOADED_LORAS_TESTS = [
),
Operation(
type=OperationType.UNLOAD,
data="Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16",
data="nvidia/llama-3.1-nemoguard-8b-topic-control",
),
Operation(
type=OperationType.LOAD,
@@ -654,7 +639,7 @@ MAX_LOADED_LORAS_TESTS = [
type=OperationType.FORWARD,
data=create_batch_data(
[
"Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16",
"nvidia/llama-3.1-nemoguard-8b-topic-control",
"philschmid/code-llama-3-1-8b-text-to-sql-lora",
]
),
@@ -672,7 +657,7 @@ MAX_LOADED_LORAS_TESTS = [
max_loaded_loras=2,
all_adapters=[
"philschmid/code-llama-3-1-8b-text-to-sql-lora",
"Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16",
"nvidia/llama-3.1-nemoguard-8b-topic-control",
"pbevan11/llama-3.1-8b-ocr-correction",
],
initial_adapters=[
@@ -685,22 +670,22 @@ MAX_LOADED_LORAS_TESTS = [
op_sequence=[
Operation(
type=OperationType.LOAD,
data="Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16",
data="nvidia/llama-3.1-nemoguard-8b-topic-control",
),
Operation(
type=OperationType.LOAD,
data="pbevan11/llama-3.1-8b-ocr-correction",
expected_implicit_evictions={
"Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16"
"nvidia/llama-3.1-nemoguard-8b-topic-control"
},
),
# Implicitly load "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16"
# Implicitly load "nvidia/llama-3.1-nemoguard-8b-topic-control"
Operation(
type=OperationType.FORWARD,
data=create_batch_data(
[
"philschmid/code-llama-3-1-8b-text-to-sql-lora",
"Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16",
"nvidia/llama-3.1-nemoguard-8b-topic-control",
]
),
expected_implicit_evictions={"pbevan11/llama-3.1-8b-ocr-correction"},
@@ -730,7 +715,7 @@ MAX_LOADED_LORAS_TESTS = [
type=OperationType.FORWARD,
data=create_batch_data(
[
"Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16",
"nvidia/llama-3.1-nemoguard-8b-topic-control",
"pbevan11/llama-3.1-8b-ocr-correction",
]
),
@@ -745,7 +730,7 @@ EVICTION_TESTS = [
max_loras_per_batch=2,
all_adapters=[
"lora1=philschmid/code-llama-3-1-8b-text-to-sql-lora",
"lora2=Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16",
"lora2=nvidia/llama-3.1-nemoguard-8b-topic-control",
"lora3=pbevan11/llama-3.1-8b-ocr-correction",
],
enable_lora=True,
@@ -764,7 +749,7 @@ EVICTION_TESTS = [
type=OperationType.LOAD,
data={
"lora_name": "lora2",
"lora_path": "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16",
"lora_path": "nvidia/llama-3.1-nemoguard-8b-topic-control",
"pinned": True,
},
expected_error="starvation",
@@ -773,7 +758,7 @@ EVICTION_TESTS = [
type=OperationType.LOAD,
data={
"lora_name": "lora2",
"lora_path": "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16",
"lora_path": "nvidia/llama-3.1-nemoguard-8b-topic-control",
"pinned": False,
},
),
@@ -1492,7 +1477,7 @@ class TestLoRADynamicUpdate(CustomTestCase):
"""
adapters = [
"philschmid/code-llama-3-1-8b-text-to-sql-lora",
"Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16",
"nvidia/llama-3.1-nemoguard-8b-topic-control",
]
with LoRAUpdateTestSession(

10
test/registered/perf/test_bench_serving_1gpu_part1.py
View File

@@ -141,9 +141,6 @@ class TestBenchServing1GPUPart1(CustomTestCase):
self.assertLess(res["median_ttft_ms"], 86)
self.assertLess(res["median_itl_ms"], 10)
@unittest.skip(
"Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16 removed from HuggingFace"
)
def test_lora_online_latency(self):
if is_in_amd_ci():
pass
@@ -159,9 +156,6 @@ class TestBenchServing1GPUPart1(CustomTestCase):
self.assertLess(res["median_e2e_latency_ms"], 2400)
self.assertLess(res["median_ttft_ms"], 58)
@unittest.skip(
"Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16 removed from HuggingFace"
)
def test_lora_online_latency_with_concurrent_adapter_updates(self):
if is_in_amd_ci():
pass
@@ -247,14 +241,14 @@ class TestBenchServing1GPUPart1(CustomTestCase):
"--mem-fraction-static",
"0.8",
"--lora-paths",
"Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16",
"nvidia/llama-3.1-nemoguard-8b-topic-control",
"--max-lora-rank",
"256",
],
dataset_name="random",
random_input_len=256,
random_output_len=256,
lora_name=["Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16"],
lora_name=["nvidia/llama-3.1-nemoguard-8b-topic-control"],
background_task=background_task,
)