feat: support external custom models (#13429)

Co-authored-by: qiuxuan.lzw <qiuxuan.lzw@alibaba-inc.com>
Co-authored-by: Mick <mickjagger19@icloud.com>

python/sglang/srt/configs/model_config.py

@@ -959,6 +959,9 @@ multimodal_model_archs = [
     "JetVLMForConditionalGeneration",
 ]
 
+if envs.SGLANG_EXTERNAL_MM_MODEL_ARCH.value:
+    multimodal_model_archs.append(envs.SGLANG_EXTERNAL_MM_MODEL_ARCH.value)
+
 
 def is_multimodal_model(model_architectures: List[str]):
     if any(

python/sglang/srt/environ.py

@@ -306,6 +306,11 @@ class Envs:
     # Health Check
     SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION = EnvBool(True)
 
+    # External models
+    SGLANG_EXTERNAL_MODEL_PACKAGE = EnvStr("")
+    SGLANG_EXTERNAL_MM_MODEL_ARCH = EnvStr("")
+    SGLANG_EXTERNAL_MM_PROCESSOR_PACKAGE = EnvStr("")
+
     # fmt: on
 
 

python/sglang/srt/managers/multimodal_processor.py

@@ -12,7 +12,7 @@ logger = logging.getLogger(__name__)
 PROCESSOR_MAPPING = {}
 
 
-def import_processors(package_name: str):
+def import_processors(package_name: str, overwrite: bool = False):
     package = importlib.import_module(package_name)
     for _, name, ispkg in pkgutil.iter_modules(package.__path__, package_name + "."):
         if not ispkg:
@@ -32,6 +32,11 @@ def import_processors(package_name: str):
             ):
                 assert hasattr(cls, "models")
                 for arch in getattr(cls, "models"):
+                    if overwrite:
+                        for model_cls, processor_cls in PROCESSOR_MAPPING.items():
+                            if model_cls.__name__ == arch.__name__:
+                                del PROCESSOR_MAPPING[model_cls]
+                                break
                     PROCESSOR_MAPPING[arch] = cls
 
 

python/sglang/srt/managers/tokenizer_manager.py

@@ -41,6 +41,7 @@ from fastapi import BackgroundTasks
 
 from sglang.srt.configs.model_config import ModelConfig
 from sglang.srt.disaggregation.utils import DisaggregationMode
+from sglang.srt.environ import envs
 from sglang.srt.lora.lora_registry import LoRARegistry
 from sglang.srt.managers.async_dynamic_batch_tokenizer import AsyncDynamicbatchTokenizer
 from sglang.srt.managers.async_mm_data_processor import AsyncMMDataProcessor
@@ -210,6 +211,10 @@ class TokenizerManager(TokenizerCommunicatorMixin):
         # Initialize tokenizer and processor
         if self.model_config.is_multimodal:
             import_processors("sglang.srt.multimodal.processors")
+            if envs.SGLANG_EXTERNAL_MM_PROCESSOR_PACKAGE.value:
+                import_processors(
+                    envs.SGLANG_EXTERNAL_MM_PROCESSOR_PACKAGE.value, overwrite=True
+                )
             try:
                 _processor = get_processor(
                     server_args.tokenizer_path,

python/sglang/srt/models/registry.py

@@ -123,3 +123,6 @@ def import_model_classes(package_name: str):
 
 ModelRegistry = _ModelRegistry()
 ModelRegistry.register("sglang.srt.models")
+
+if envs.SGLANG_EXTERNAL_MODEL_PACKAGE.value:
+    ModelRegistry.register(envs.SGLANG_EXTERNAL_MODEL_PACKAGE.value, overwrite=True)