From 963def7f26a9595e4713756e07aed4db12a72727 Mon Sep 17 00:00:00 2001 From: satyamk7054 <43010011+satyamk7054@users.noreply.github.com> Date: Thu, 19 Feb 2026 05:03:19 -0800 Subject: [PATCH] Move lora request validation to tokenizer_manager from server (#18962) Co-authored-by: Satyam Kumar --- .../srt/entrypoints/openai/serving_base.py | 13 ---- .../srt/entrypoints/openai/serving_chat.py | 9 --- .../entrypoints/openai/serving_completions.py | 8 -- .../entrypoints/openai/serving_embedding.py | 8 -- .../sglang/srt/managers/tokenizer_manager.py | 24 +++++- .../lora/test_embedding_lora_support.py | 18 +++++ test/registered/lora/test_lora_openai_api.py | 74 ------------------- 7 files changed, 40 insertions(+), 114 deletions(-) diff --git a/python/sglang/srt/entrypoints/openai/serving_base.py b/python/sglang/srt/entrypoints/openai/serving_base.py index 6e01d2fd0..8fed454a7 100644 --- a/python/sglang/srt/entrypoints/openai/serving_base.py +++ b/python/sglang/srt/entrypoints/openai/serving_base.py @@ -70,19 +70,6 @@ class OpenAIServingBase(ABC): # Fall back to explicit lora_path return explicit_lora_path - def _validate_lora_enabled(self, adapter_name: str) -> None: - """Check that LoRA is enabled before attempting to use an adapter. - - Raises ValueError with actionable guidance if --enable-lora flag is missing. - Adapter existence is validated later by TokenizerManager.lora_registry. - """ - if not self.tokenizer_manager.server_args.enable_lora: - raise ValueError( - f"LoRA adapter '{adapter_name}' was requested, but LoRA is not enabled. " - "Please launch the server with --enable-lora flag and preload adapters " - "using --lora-paths or /load_lora_adapter endpoint." - ) - async def handle_request( self, request: OpenAIServingRequest, raw_request: Request ) -> Union[Any, StreamingResponse, ErrorResponse]: diff --git a/python/sglang/srt/entrypoints/openai/serving_chat.py b/python/sglang/srt/entrypoints/openai/serving_chat.py index cb319fe5f..3b230fd04 100644 --- a/python/sglang/srt/entrypoints/openai/serving_chat.py +++ b/python/sglang/srt/entrypoints/openai/serving_chat.py @@ -277,15 +277,6 @@ class OpenAIServingChat(OpenAIServingBase): # Resolve LoRA adapter from model parameter or explicit lora_path lora_path = self._resolve_lora_path(request.model, request.lora_path) - if lora_path: - first_adapter = ( - lora_path - if isinstance(lora_path, str) - else next((a for a in lora_path if a), None) - ) - if first_adapter: - self._validate_lora_enabled(first_adapter) - img_max_dynamic_patch, vid_max_dynamic_patch = _extract_max_dynamic_patch( request ) diff --git a/python/sglang/srt/entrypoints/openai/serving_completions.py b/python/sglang/srt/entrypoints/openai/serving_completions.py index dee22ef18..f071027f0 100644 --- a/python/sglang/srt/entrypoints/openai/serving_completions.py +++ b/python/sglang/srt/entrypoints/openai/serving_completions.py @@ -98,14 +98,6 @@ class OpenAIServingCompletion(OpenAIServingBase): # Resolve LoRA adapter from model parameter or explicit lora_path lora_path = self._resolve_lora_path(request.model, request.lora_path) - if lora_path: - first_adapter = ( - lora_path - if isinstance(lora_path, str) - else next((a for a in lora_path if a), None) - ) - if first_adapter: - self._validate_lora_enabled(first_adapter) adapted_request = GenerateReqInput( **prompt_kwargs, diff --git a/python/sglang/srt/entrypoints/openai/serving_embedding.py b/python/sglang/srt/entrypoints/openai/serving_embedding.py index bd32f556d..7555cc476 100644 --- a/python/sglang/srt/entrypoints/openai/serving_embedding.py +++ b/python/sglang/srt/entrypoints/openai/serving_embedding.py @@ -128,14 +128,6 @@ class OpenAIServingEmbedding(OpenAIServingBase): # Resolve LoRA adapter from model parameter or explicit lora_path lora_path = self._resolve_lora_path(request.model, request.lora_path) - if lora_path: - first_adapter = ( - lora_path - if isinstance(lora_path, str) - else next((a for a in lora_path if a), None) - ) - if first_adapter: - self._validate_lora_enabled(first_adapter) adapted_request = EmbeddingReqInput( **prompt_kwargs, diff --git a/python/sglang/srt/managers/tokenizer_manager.py b/python/sglang/srt/managers/tokenizer_manager.py index 5bbbd5ed0..0914a5230 100644 --- a/python/sglang/srt/managers/tokenizer_manager.py +++ b/python/sglang/srt/managers/tokenizer_manager.py @@ -511,8 +511,7 @@ class TokenizerManager(TokenizerCommunicatorMixin, TokenizerManagerMultiItemMixi await self.is_pause_cond.wait_for(lambda: not self.is_pause) async with self.model_update_lock.reader_lock: - if self.server_args.enable_lora and obj.lora_path: - await self._resolve_lora_path(obj) + await self._validate_and_resolve_lora(obj) # Tokenize the request and send it to the scheduler if obj.is_single: @@ -2213,6 +2212,27 @@ class TokenizerManager(TokenizerCommunicatorMixin, TokenizerManagerMultiItemMixi if len(self.model_update_tmp) == self.server_args.dp_size: self.model_update_result.set_result(self.model_update_tmp) + async def _validate_and_resolve_lora( + self, obj: Union[GenerateReqInput, EmbeddingReqInput] + ) -> None: + if not obj.lora_path: + return + + if not self.server_args.enable_lora: + first_adapter = ( + obj.lora_path + if isinstance(obj.lora_path, str) + else next((a for a in obj.lora_path if a), None) + ) + + raise ValueError( + f"LoRA adapter '{first_adapter}' was requested, but LoRA is not enabled. " + "Please launch the server with --enable-lora flag and preload adapters " + "using --lora-paths or /load_lora_adapter endpoint." + ) + + await self._resolve_lora_path(obj) + async def _resolve_lora_path(self, obj: Union[GenerateReqInput, EmbeddingReqInput]): if isinstance(obj.lora_path, str): unique_lora_paths = set([obj.lora_path]) diff --git a/test/registered/lora/test_embedding_lora_support.py b/test/registered/lora/test_embedding_lora_support.py index 2fa69b186..32f397c3a 100644 --- a/test/registered/lora/test_embedding_lora_support.py +++ b/test/registered/lora/test_embedding_lora_support.py @@ -46,6 +46,24 @@ register_cuda_ci( class TestEmbeddingLoraSupport(unittest.TestCase): """Test LoRA support in embedding request structures.""" + def test_engine_encode_validates_enable_lora(self): + """Test Engine.encode() validates enable_lora before processing lora_path.""" + # Use a simple non-gated model for this validation test + with SRTRunner( + MODEL_PATH, + torch_dtype=torch.float16, + model_type="embedding", + port=DEFAULT_PORT_FOR_SRT_TEST_RUNNER, + ) as runner: + # Should raise ValueError because enable_lora was not set for the server + with self.assertRaises(ValueError) as context: + runner.engine.encode(prompt="Test", lora_path="fake-adapter") + + error_msg = str(context.exception) + self.assertIn("not enabled", error_msg.lower()) + self.assertIn("--enable-lora", error_msg) + self.assertIn("fake-adapter", error_msg) + def test_embedding_lora_fields(self): """Test LoRA fields exist and work correctly across all embedding structures.""" # EmbeddingReqInput: fields exist, normalization expands single to batch, indexing works diff --git a/test/registered/lora/test_lora_openai_api.py b/test/registered/lora/test_lora_openai_api.py index 140d95816..69b2e6e3c 100644 --- a/test/registered/lora/test_lora_openai_api.py +++ b/test/registered/lora/test_lora_openai_api.py @@ -142,44 +142,6 @@ class TestResolveLoraPath(unittest.TestCase): self.assertEqual(result, "adapter-name") -class TestValidateLoraEnabled(unittest.TestCase): - """Test _validate_lora_enabled method.""" - - def test_validation_passes_when_lora_enabled(self): - """Test validation passes when LoRA is enabled.""" - tokenizer_manager = MockTokenizerManager(enable_lora=True) - serving = ConcreteServingBase(tokenizer_manager) - - # Should not raise - try: - serving._validate_lora_enabled("sql-expert") - except ValueError: - self.fail("_validate_lora_enabled raised ValueError unexpectedly") - - def test_validation_fails_when_lora_disabled(self): - """Test validation fails with helpful message when LoRA is disabled.""" - tokenizer_manager = MockTokenizerManager(enable_lora=False) - serving = ConcreteServingBase(tokenizer_manager) - - with self.assertRaises(ValueError) as context: - serving._validate_lora_enabled("sql-expert") - - error_message = str(context.exception) - self.assertIn("sql-expert", error_message) - self.assertIn("--enable-lora", error_message) - self.assertIn("not enabled", error_message) - - def test_validation_error_mentions_adapter_name(self): - """Test that error message includes the requested adapter name.""" - tokenizer_manager = MockTokenizerManager(enable_lora=False) - serving = ConcreteServingBase(tokenizer_manager) - - with self.assertRaises(ValueError) as context: - serving._validate_lora_enabled("my-custom-adapter") - - self.assertIn("my-custom-adapter", str(context.exception)) - - class TestIntegrationScenarios(unittest.TestCase): """Integration tests for common usage scenarios.""" @@ -196,9 +158,6 @@ class TestIntegrationScenarios(unittest.TestCase): lora_path = self.serving._resolve_lora_path(model, explicit_lora) self.assertEqual(lora_path, "sql-expert") - # Validation should pass - self.serving._validate_lora_enabled(lora_path) - def test_backward_compatible_usage(self): """Test backward-compatible usage with explicit lora_path.""" model = "meta-llama/Llama-3.1-8B" @@ -207,9 +166,6 @@ class TestIntegrationScenarios(unittest.TestCase): lora_path = self.serving._resolve_lora_path(model, explicit_lora) self.assertEqual(lora_path, "sql-expert") - # Validation should pass - self.serving._validate_lora_enabled(lora_path) - def test_base_model_usage(self): """Test using base model without any adapter.""" model = "meta-llama/Llama-3.1-8B" @@ -228,10 +184,6 @@ class TestIntegrationScenarios(unittest.TestCase): lora_path = self.serving._resolve_lora_path(model, explicit_lora) self.assertEqual(lora_path, explicit_lora) - # Validate first adapter in list - if isinstance(lora_path, list) and lora_path[0]: - self.serving._validate_lora_enabled(lora_path[0]) - def test_adapter_in_model_overrides_batch_list(self): """Test that adapter in model parameter overrides batch list.""" model = "meta-llama/Llama-3.1-8B:preferred-adapter" @@ -240,24 +192,6 @@ class TestIntegrationScenarios(unittest.TestCase): lora_path = self.serving._resolve_lora_path(model, explicit_lora) self.assertEqual(lora_path, "preferred-adapter") - def test_error_when_lora_not_enabled(self): - """Test comprehensive error flow when LoRA is not enabled.""" - # Setup server without LoRA enabled - tokenizer_manager = MockTokenizerManager(enable_lora=False) - serving = ConcreteServingBase(tokenizer_manager) - - # User tries to use adapter - model = "meta-llama/Llama-3.1-8B:sql-expert" - lora_path = serving._resolve_lora_path(model, None) - - # Should get helpful error - with self.assertRaises(ValueError) as context: - serving._validate_lora_enabled(lora_path) - - error = str(context.exception) - self.assertIn("--enable-lora", error) - self.assertIn("sql-expert", error) - class TestEdgeCases(unittest.TestCase): """Test edge cases and error conditions.""" @@ -318,14 +252,6 @@ class TestEdgeCases(unittest.TestCase): result = self.serving._resolve_lora_path("model-name", "") self.assertEqual(result, "") - def test_validation_with_empty_adapter_name(self): - """Test validation with empty adapter name still raises error.""" - tokenizer_manager = MockTokenizerManager(enable_lora=False) - serving = ConcreteServingBase(tokenizer_manager) - - with self.assertRaises(ValueError): - serving._validate_lora_enabled("") - if __name__ == "__main__": unittest.main()