diff --git a/python/sglang/srt/configs/model_config.py b/python/sglang/srt/configs/model_config.py index ba0317bfe..446533f53 100644 --- a/python/sglang/srt/configs/model_config.py +++ b/python/sglang/srt/configs/model_config.py @@ -959,6 +959,9 @@ multimodal_model_archs = [ "JetVLMForConditionalGeneration", ] +if envs.SGLANG_EXTERNAL_MM_MODEL_ARCH.value: + multimodal_model_archs.append(envs.SGLANG_EXTERNAL_MM_MODEL_ARCH.value) + def is_multimodal_model(model_architectures: List[str]): if any( diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py index 72d87dfa0..63ce4b0a8 100644 --- a/python/sglang/srt/environ.py +++ b/python/sglang/srt/environ.py @@ -306,6 +306,11 @@ class Envs: # Health Check SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION = EnvBool(True) + # External models + SGLANG_EXTERNAL_MODEL_PACKAGE = EnvStr("") + SGLANG_EXTERNAL_MM_MODEL_ARCH = EnvStr("") + SGLANG_EXTERNAL_MM_PROCESSOR_PACKAGE = EnvStr("") + # fmt: on diff --git a/python/sglang/srt/managers/multimodal_processor.py b/python/sglang/srt/managers/multimodal_processor.py index 7826241d0..3b15e566f 100644 --- a/python/sglang/srt/managers/multimodal_processor.py +++ b/python/sglang/srt/managers/multimodal_processor.py @@ -12,7 +12,7 @@ logger = logging.getLogger(__name__) PROCESSOR_MAPPING = {} -def import_processors(package_name: str): +def import_processors(package_name: str, overwrite: bool = False): package = importlib.import_module(package_name) for _, name, ispkg in pkgutil.iter_modules(package.__path__, package_name + "."): if not ispkg: @@ -32,6 +32,11 @@ def import_processors(package_name: str): ): assert hasattr(cls, "models") for arch in getattr(cls, "models"): + if overwrite: + for model_cls, processor_cls in PROCESSOR_MAPPING.items(): + if model_cls.__name__ == arch.__name__: + del PROCESSOR_MAPPING[model_cls] + break PROCESSOR_MAPPING[arch] = cls diff --git a/python/sglang/srt/managers/tokenizer_manager.py b/python/sglang/srt/managers/tokenizer_manager.py index 3375ab60e..803fbdce0 100644 --- a/python/sglang/srt/managers/tokenizer_manager.py +++ b/python/sglang/srt/managers/tokenizer_manager.py @@ -41,6 +41,7 @@ from fastapi import BackgroundTasks from sglang.srt.configs.model_config import ModelConfig from sglang.srt.disaggregation.utils import DisaggregationMode +from sglang.srt.environ import envs from sglang.srt.lora.lora_registry import LoRARegistry from sglang.srt.managers.async_dynamic_batch_tokenizer import AsyncDynamicbatchTokenizer from sglang.srt.managers.async_mm_data_processor import AsyncMMDataProcessor @@ -210,6 +211,10 @@ class TokenizerManager(TokenizerCommunicatorMixin): # Initialize tokenizer and processor if self.model_config.is_multimodal: import_processors("sglang.srt.multimodal.processors") + if envs.SGLANG_EXTERNAL_MM_PROCESSOR_PACKAGE.value: + import_processors( + envs.SGLANG_EXTERNAL_MM_PROCESSOR_PACKAGE.value, overwrite=True + ) try: _processor = get_processor( server_args.tokenizer_path, diff --git a/python/sglang/srt/models/registry.py b/python/sglang/srt/models/registry.py index 1745b7d7b..76cb5b640 100644 --- a/python/sglang/srt/models/registry.py +++ b/python/sglang/srt/models/registry.py @@ -123,3 +123,6 @@ def import_model_classes(package_name: str): ModelRegistry = _ModelRegistry() ModelRegistry.register("sglang.srt.models") + +if envs.SGLANG_EXTERNAL_MODEL_PACKAGE.value: + ModelRegistry.register(envs.SGLANG_EXTERNAL_MODEL_PACKAGE.value, overwrite=True) diff --git a/test/srt/external_models/custom_qwen2_vl.py b/test/srt/external_models/custom_qwen2_vl.py new file mode 100644 index 000000000..5e1b96496 --- /dev/null +++ b/test/srt/external_models/custom_qwen2_vl.py @@ -0,0 +1,21 @@ +from sglang.srt.models.qwen2_vl import ( + Qwen2VLForConditionalGeneration as OriginalQwen2VLForConditionalGeneration, +) +from sglang.srt.multimodal.processors.qwen_vl import QwenVLImageProcessor + + +class Qwen2VLForConditionalGeneration(OriginalQwen2VLForConditionalGeneration): + def __init__(self, config, quant_config, prefix: str = "") -> None: + super().__init__(config, quant_config, prefix) + print("init custom model:", self.__class__.__name__) + + +class CustomProcessor(QwenVLImageProcessor): + models = [Qwen2VLForConditionalGeneration] + + def __init__(self, hf_config, server_args, _processor, *args, **kwargs): + super().__init__(hf_config, server_args, _processor, *args, **kwargs) + print("init custom processor:", self.__class__.__name__) + + +EntryClass = Qwen2VLForConditionalGeneration diff --git a/test/srt/run_suite.py b/test/srt/run_suite.py index 844391357..1bcde43a1 100644 --- a/test/srt/run_suite.py +++ b/test/srt/run_suite.py @@ -65,6 +65,7 @@ suites = { TestFile("test_eagle_infer_beta.py", 90), TestFile("test_constrained_decoding.py", 150), TestFile("test_eval_fp8_accuracy.py", 303), + TestFile("test_external_models.py", 155), TestFile("test_fa3.py", 420), TestFile("test_flashmla.py", 230), TestFile("test_fp8_utils.py", 5), diff --git a/test/srt/test_external_models.py b/test/srt/test_external_models.py new file mode 100644 index 000000000..cab5b785a --- /dev/null +++ b/test/srt/test_external_models.py @@ -0,0 +1,28 @@ +import os +import unittest + +import sglang as sgl +from sglang.test.test_utils import CustomTestCase + + +class TestExternalModels(CustomTestCase): + def test_external_model(self): + os.environ["SGLANG_EXTERNAL_MODEL_PACKAGE"] = "external_models" + os.environ["SGLANG_EXTERNAL_MM_PROCESSOR_PACKAGE"] = "external_models" + prompt = "Today is a sunny day and I like" + model_path = "Qwen/Qwen2-VL-2B-Instruct" + + engine = sgl.Engine( + model_path=model_path, + cuda_graph_max_bs=1, + max_total_tokens=64, + enable_multimodal=True, + ) + out = engine.generate(prompt)["text"] + engine.shutdown() + + self.assertGreater(len(out), 0) + + +if __name__ == "__main__": + unittest.main()