From cc860a2198665dfd5d433c9145ba0573fe73b132 Mon Sep 17 00:00:00 2001 From: Glen Liu <62917497+glenliu21@users.noreply.github.com> Date: Mon, 2 Mar 2026 15:55:41 -0500 Subject: [PATCH] [TestFix] change LoRA tests to use NVIDIA adapter instead of Nutanix (#19642) Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- python/sglang/test/lora_utils.py | 4 +- test/registered/lora/test_lora_update.py | 127 ++++++++---------- .../perf/test_bench_serving_1gpu_part1.py | 10 +- 3 files changed, 60 insertions(+), 81 deletions(-) diff --git a/python/sglang/test/lora_utils.py b/python/sglang/test/lora_utils.py index b1181f248..ad2b9f429 100644 --- a/python/sglang/test/lora_utils.py +++ b/python/sglang/test/lora_utils.py @@ -68,7 +68,7 @@ ALL_OTHER_LORA_MODELS = [ base="meta-llama/Llama-3.1-8B-Instruct", adaptors=[ LoRAAdaptor( - name="Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + name="nvidia/llama-3.1-nemoguard-8b-topic-control", prefill_tolerance=1e-1, ), ], @@ -109,7 +109,7 @@ ALL_OTHER_MULTI_LORA_MODELS = [ prefill_tolerance=1e-1, ), LoRAAdaptor( - name="Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + name="nvidia/llama-3.1-nemoguard-8b-topic-control", prefill_tolerance=1e-1, ), ], diff --git a/test/registered/lora/test_lora_update.py b/test/registered/lora/test_lora_update.py index 70f5c3b04..9b90881ae 100644 --- a/test/registered/lora/test_lora_update.py +++ b/test/registered/lora/test_lora_update.py @@ -37,7 +37,6 @@ from sglang.test.test_utils import ( register_cuda_ci( est_time=487, suite="stage-b-test-large-1-gpu", - disabled="Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16 removed from HuggingFace", ) PROMPTS = [ @@ -98,13 +97,13 @@ BASIC_TESTS = [ max_loras_per_batch=3, all_adapters=[ "philschmid/code-llama-3-1-8b-text-to-sql-lora", - "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + "nvidia/llama-3.1-nemoguard-8b-topic-control", "pbevan11/llama-3.1-8b-ocr-correction", ], initial_adapters=[ # Testing 3 supported lora-path formats. "philschmid/code-llama-3-1-8b-text-to-sql-lora", - "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16=Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + "nvidia/llama-3.1-nemoguard-8b-topic-control=nvidia/llama-3.1-nemoguard-8b-topic-control", { "lora_name": "pbevan11/llama-3.1-8b-ocr-correction", "lora_path": "pbevan11/llama-3.1-8b-ocr-correction", @@ -130,14 +129,14 @@ BASIC_TESTS = [ data=create_batch_data( [ "philschmid/code-llama-3-1-8b-text-to-sql-lora", - "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + "nvidia/llama-3.1-nemoguard-8b-topic-control", "pbevan11/llama-3.1-8b-ocr-correction", ] ), ), Operation( type=OperationType.UNLOAD, - data="Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + data="nvidia/llama-3.1-nemoguard-8b-topic-control", ), Operation( type=OperationType.UNLOAD, @@ -149,9 +148,7 @@ BASIC_TESTS = [ ), Operation( type=OperationType.FORWARD, - data=create_batch_data( - "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16" - ), + data=create_batch_data("nvidia/llama-3.1-nemoguard-8b-topic-control"), ), Operation( type=OperationType.FORWARD, @@ -159,7 +156,7 @@ BASIC_TESTS = [ ), Operation( type=OperationType.LOAD, - data="Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + data="nvidia/llama-3.1-nemoguard-8b-topic-control", expected_error="already loaded", ), Operation( @@ -172,7 +169,7 @@ BASIC_TESTS = [ data=create_batch_data( [ "philschmid/code-llama-3-1-8b-text-to-sql-lora", - "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + "nvidia/llama-3.1-nemoguard-8b-topic-control", "pbevan11/llama-3.1-8b-ocr-correction", ] ), @@ -189,14 +186,14 @@ BASIC_TESTS = [ type=OperationType.FORWARD, data=create_batch_data( [ - "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + "nvidia/llama-3.1-nemoguard-8b-topic-control", "pbevan11/llama-3.1-8b-ocr-correction", ] ), ), Operation( type=OperationType.UNLOAD, - data="Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + data="nvidia/llama-3.1-nemoguard-8b-topic-control", ), Operation( type=OperationType.UNLOAD, @@ -204,9 +201,7 @@ BASIC_TESTS = [ ), Operation( type=OperationType.FORWARD, - data=create_batch_data( - "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16" - ), + data=create_batch_data("nvidia/llama-3.1-nemoguard-8b-topic-control"), ), Operation( type=OperationType.FORWARD, @@ -229,7 +224,7 @@ BASIC_TESTS = [ max_loras_per_batch=4, all_adapters=[ "philschmid/code-llama-3-1-8b-text-to-sql-lora", - "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + "nvidia/llama-3.1-nemoguard-8b-topic-control", "pbevan11/llama-3.1-8b-ocr-correction", ], op_sequence=[ @@ -239,7 +234,7 @@ BASIC_TESTS = [ ), Operation( type=OperationType.LOAD, - data="Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + data="nvidia/llama-3.1-nemoguard-8b-topic-control", ), Operation( type=OperationType.LOAD, @@ -263,7 +258,7 @@ BASIC_TESTS = [ data=create_batch_data( [ "philschmid/code-llama-3-1-8b-text-to-sql-lora", - "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + "nvidia/llama-3.1-nemoguard-8b-topic-control", "pbevan11/llama-3.1-8b-ocr-correction", None, ] @@ -282,7 +277,7 @@ BASIC_TESTS = [ data=create_batch_data( [ None, - "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + "nvidia/llama-3.1-nemoguard-8b-topic-control", "pbevan11/llama-3.1-8b-ocr-correction", None, ] @@ -290,7 +285,7 @@ BASIC_TESTS = [ ), Operation( type=OperationType.UNLOAD, - data="Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + data="nvidia/llama-3.1-nemoguard-8b-topic-control", ), Operation( type=OperationType.UNLOAD, @@ -298,9 +293,7 @@ BASIC_TESTS = [ ), Operation( type=OperationType.FORWARD, - data=create_batch_data( - "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16" - ), + data=create_batch_data("nvidia/llama-3.1-nemoguard-8b-topic-control"), ), Operation( type=OperationType.FORWARD, @@ -317,7 +310,7 @@ BASIC_TESTS = [ ), Operation( type=OperationType.LOAD, - data="Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + data="nvidia/llama-3.1-nemoguard-8b-topic-control", expected_error="already loaded", ), Operation( @@ -330,7 +323,7 @@ BASIC_TESTS = [ data=create_batch_data( [ "philschmid/code-llama-3-1-8b-text-to-sql-lora", - "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + "nvidia/llama-3.1-nemoguard-8b-topic-control", "pbevan11/llama-3.1-8b-ocr-correction", None, ] @@ -347,7 +340,7 @@ TARGET_MODULE_TESTS = [ lora_target_modules=["all"], max_lora_rank=64, all_adapters=[ - "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", # target_modules = q, k, v, o, gate, up, down + "nvidia/llama-3.1-nemoguard-8b-topic-control", # target_modules = q, k, v, o, gate, up, down "algoprog/fact-generation-llama-3.1-8b-instruct-lora", # target_modules = q, k, v, o, gate ], initial_adapters=["algoprog/fact-generation-llama-3.1-8b-instruct-lora"], @@ -360,21 +353,19 @@ TARGET_MODULE_TESTS = [ ), Operation( type=OperationType.FORWARD, - data=create_batch_data( - "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16" - ), + data=create_batch_data("nvidia/llama-3.1-nemoguard-8b-topic-control"), expected_error="never been loaded", ), Operation( type=OperationType.LOAD, - data="Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + data="nvidia/llama-3.1-nemoguard-8b-topic-control", ), Operation( type=OperationType.FORWARD, data=create_batch_data( [ "algoprog/fact-generation-llama-3.1-8b-instruct-lora", - "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + "nvidia/llama-3.1-nemoguard-8b-topic-control", None, ] ), @@ -387,16 +378,14 @@ TARGET_MODULE_TESTS = [ max_loras_per_batch=3, max_lora_rank=64, all_adapters=[ - "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", # target_modules = q, k, v, o, gate, up, down + "nvidia/llama-3.1-nemoguard-8b-topic-control", # target_modules = q, k, v, o, gate, up, down "algoprog/fact-generation-llama-3.1-8b-instruct-lora", # target_modules = q, k, v, o, gate ], - initial_adapters=["Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16"], + initial_adapters=["nvidia/llama-3.1-nemoguard-8b-topic-control"], op_sequence=[ Operation( type=OperationType.FORWARD, - data=create_batch_data( - "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16" - ), + data=create_batch_data("nvidia/llama-3.1-nemoguard-8b-topic-control"), ), Operation( type=OperationType.FORWARD, @@ -414,7 +403,7 @@ TARGET_MODULE_TESTS = [ data=create_batch_data( [ "algoprog/fact-generation-llama-3.1-8b-instruct-lora", - "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + "nvidia/llama-3.1-nemoguard-8b-topic-control", None, ] ), @@ -427,7 +416,7 @@ TARGET_MODULE_TESTS = [ max_loras_per_batch=3, max_lora_rank=64, all_adapters=[ - "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", # target_modules = q, k, v, o, gate, up, down + "nvidia/llama-3.1-nemoguard-8b-topic-control", # target_modules = q, k, v, o, gate, up, down "algoprog/fact-generation-llama-3.1-8b-instruct-lora", # target_modules = q, k, v, o, gate ], initial_adapters=["algoprog/fact-generation-llama-3.1-8b-instruct-lora"], @@ -440,14 +429,12 @@ TARGET_MODULE_TESTS = [ ), Operation( type=OperationType.FORWARD, - data=create_batch_data( - "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16" - ), + data=create_batch_data("nvidia/llama-3.1-nemoguard-8b-topic-control"), expected_error="never been loaded", ), Operation( type=OperationType.LOAD, - data="Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + data="nvidia/llama-3.1-nemoguard-8b-topic-control", expected_error="incompatible", ), Operation( @@ -469,17 +456,15 @@ MAX_LORA_RANK_TESTS = [ max_loras_per_batch=3, max_lora_rank=32, all_adapters=[ - "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", # r = 4 + "nvidia/llama-3.1-nemoguard-8b-topic-control", # r = 4 "pbevan11/llama-3.1-8b-ocr-correction", # r = 32 "philschmid/code-llama-3-1-8b-text-to-sql-lora", # r = 256 ], - initial_adapters=["Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16"], + initial_adapters=["nvidia/llama-3.1-nemoguard-8b-topic-control"], op_sequence=[ Operation( type=OperationType.FORWARD, - data=create_batch_data( - "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16" - ), + data=create_batch_data("nvidia/llama-3.1-nemoguard-8b-topic-control"), ), Operation( type=OperationType.FORWARD, @@ -500,7 +485,7 @@ MAX_LORA_RANK_TESTS = [ data=create_batch_data( [ "pbevan11/llama-3.1-8b-ocr-correction", - "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + "nvidia/llama-3.1-nemoguard-8b-topic-control", None, ] ), @@ -522,7 +507,7 @@ MAX_LORA_RANK_TESTS = [ data=create_batch_data( [ "pbevan11/llama-3.1-8b-ocr-correction", - "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + "nvidia/llama-3.1-nemoguard-8b-topic-control", None, ] ), @@ -534,7 +519,7 @@ MAX_LORA_RANK_TESTS = [ base="meta-llama/Llama-3.1-8B-Instruct", max_loras_per_batch=3, all_adapters=[ - "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", # r = 4 + "nvidia/llama-3.1-nemoguard-8b-topic-control", # r = 4 "pbevan11/llama-3.1-8b-ocr-correction", # r = 32 "philschmid/code-llama-3-1-8b-text-to-sql-lora", # r = 256 ], @@ -556,19 +541,19 @@ MAX_LORA_RANK_TESTS = [ ), Operation( type=OperationType.LOAD, - data="Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + data="nvidia/llama-3.1-nemoguard-8b-topic-control", ), Operation( type=OperationType.FORWARD, data=create_batch_data( - "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + "nvidia/llama-3.1-nemoguard-8b-topic-control", ), ), Operation( type=OperationType.FORWARD, data=create_batch_data( [ - "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + "nvidia/llama-3.1-nemoguard-8b-topic-control", "pbevan11/llama-3.1-8b-ocr-correction", None, ] @@ -585,14 +570,14 @@ MAX_LOADED_LORAS_TESTS = [ max_loaded_loras=2, all_adapters=[ "philschmid/code-llama-3-1-8b-text-to-sql-lora", - "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + "nvidia/llama-3.1-nemoguard-8b-topic-control", "pbevan11/llama-3.1-8b-ocr-correction", ], initial_adapters=["philschmid/code-llama-3-1-8b-text-to-sql-lora"], op_sequence=[ Operation( type=OperationType.LOAD, - data="Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + data="nvidia/llama-3.1-nemoguard-8b-topic-control", ), Operation( type=OperationType.LOAD, @@ -606,7 +591,7 @@ MAX_LOADED_LORAS_TESTS = [ type=OperationType.FORWARD, data=create_batch_data( [ - "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + "nvidia/llama-3.1-nemoguard-8b-topic-control", "philschmid/code-llama-3-1-8b-text-to-sql-lora", ] ), @@ -620,7 +605,7 @@ MAX_LOADED_LORAS_TESTS = [ type=OperationType.FORWARD, data=create_batch_data( [ - "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + "nvidia/llama-3.1-nemoguard-8b-topic-control", ] ), ), @@ -628,13 +613,13 @@ MAX_LOADED_LORAS_TESTS = [ type=OperationType.LOAD, data="philschmid/code-llama-3-1-8b-text-to-sql-lora", ), - # Implicitly load "pbevan11/llama-3.1-8b-ocr-correction" and make sure that "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16" + # Implicitly load "pbevan11/llama-3.1-8b-ocr-correction" and make sure that "nvidia/llama-3.1-nemoguard-8b-topic-control" # isn't implicitly unloaded even though it is LRU because it is needed for this forward pass Operation( type=OperationType.FORWARD, data=create_batch_data( [ - "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + "nvidia/llama-3.1-nemoguard-8b-topic-control", "pbevan11/llama-3.1-8b-ocr-correction", ] ), @@ -644,7 +629,7 @@ MAX_LOADED_LORAS_TESTS = [ ), Operation( type=OperationType.UNLOAD, - data="Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + data="nvidia/llama-3.1-nemoguard-8b-topic-control", ), Operation( type=OperationType.LOAD, @@ -654,7 +639,7 @@ MAX_LOADED_LORAS_TESTS = [ type=OperationType.FORWARD, data=create_batch_data( [ - "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + "nvidia/llama-3.1-nemoguard-8b-topic-control", "philschmid/code-llama-3-1-8b-text-to-sql-lora", ] ), @@ -672,7 +657,7 @@ MAX_LOADED_LORAS_TESTS = [ max_loaded_loras=2, all_adapters=[ "philschmid/code-llama-3-1-8b-text-to-sql-lora", - "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + "nvidia/llama-3.1-nemoguard-8b-topic-control", "pbevan11/llama-3.1-8b-ocr-correction", ], initial_adapters=[ @@ -685,22 +670,22 @@ MAX_LOADED_LORAS_TESTS = [ op_sequence=[ Operation( type=OperationType.LOAD, - data="Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + data="nvidia/llama-3.1-nemoguard-8b-topic-control", ), Operation( type=OperationType.LOAD, data="pbevan11/llama-3.1-8b-ocr-correction", expected_implicit_evictions={ - "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16" + "nvidia/llama-3.1-nemoguard-8b-topic-control" }, ), - # Implicitly load "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16" + # Implicitly load "nvidia/llama-3.1-nemoguard-8b-topic-control" Operation( type=OperationType.FORWARD, data=create_batch_data( [ "philschmid/code-llama-3-1-8b-text-to-sql-lora", - "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + "nvidia/llama-3.1-nemoguard-8b-topic-control", ] ), expected_implicit_evictions={"pbevan11/llama-3.1-8b-ocr-correction"}, @@ -730,7 +715,7 @@ MAX_LOADED_LORAS_TESTS = [ type=OperationType.FORWARD, data=create_batch_data( [ - "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + "nvidia/llama-3.1-nemoguard-8b-topic-control", "pbevan11/llama-3.1-8b-ocr-correction", ] ), @@ -745,7 +730,7 @@ EVICTION_TESTS = [ max_loras_per_batch=2, all_adapters=[ "lora1=philschmid/code-llama-3-1-8b-text-to-sql-lora", - "lora2=Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + "lora2=nvidia/llama-3.1-nemoguard-8b-topic-control", "lora3=pbevan11/llama-3.1-8b-ocr-correction", ], enable_lora=True, @@ -764,7 +749,7 @@ EVICTION_TESTS = [ type=OperationType.LOAD, data={ "lora_name": "lora2", - "lora_path": "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + "lora_path": "nvidia/llama-3.1-nemoguard-8b-topic-control", "pinned": True, }, expected_error="starvation", @@ -773,7 +758,7 @@ EVICTION_TESTS = [ type=OperationType.LOAD, data={ "lora_name": "lora2", - "lora_path": "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + "lora_path": "nvidia/llama-3.1-nemoguard-8b-topic-control", "pinned": False, }, ), @@ -1492,7 +1477,7 @@ class TestLoRADynamicUpdate(CustomTestCase): """ adapters = [ "philschmid/code-llama-3-1-8b-text-to-sql-lora", - "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + "nvidia/llama-3.1-nemoguard-8b-topic-control", ] with LoRAUpdateTestSession( diff --git a/test/registered/perf/test_bench_serving_1gpu_part1.py b/test/registered/perf/test_bench_serving_1gpu_part1.py index 27cdcf156..385647efc 100644 --- a/test/registered/perf/test_bench_serving_1gpu_part1.py +++ b/test/registered/perf/test_bench_serving_1gpu_part1.py @@ -141,9 +141,6 @@ class TestBenchServing1GPUPart1(CustomTestCase): self.assertLess(res["median_ttft_ms"], 86) self.assertLess(res["median_itl_ms"], 10) - @unittest.skip( - "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16 removed from HuggingFace" - ) def test_lora_online_latency(self): if is_in_amd_ci(): pass @@ -159,9 +156,6 @@ class TestBenchServing1GPUPart1(CustomTestCase): self.assertLess(res["median_e2e_latency_ms"], 2400) self.assertLess(res["median_ttft_ms"], 58) - @unittest.skip( - "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16 removed from HuggingFace" - ) def test_lora_online_latency_with_concurrent_adapter_updates(self): if is_in_amd_ci(): pass @@ -247,14 +241,14 @@ class TestBenchServing1GPUPart1(CustomTestCase): "--mem-fraction-static", "0.8", "--lora-paths", - "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + "nvidia/llama-3.1-nemoguard-8b-topic-control", "--max-lora-rank", "256", ], dataset_name="random", random_input_len=256, random_output_len=256, - lora_name=["Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16"], + lora_name=["nvidia/llama-3.1-nemoguard-8b-topic-control"], background_task=background_task, )