Unify the model type checking (#1905)

python/sglang/srt/managers/image_processor.py

@@ -180,7 +180,7 @@ class LlavaImageProcessor(BaseImageProcessor):
             "pixel_values": pixel_values,
             "image_hashes": image_hashes,
             "image_sizes": image_sizes,
-            "modalities": request_obj.modalities,
+            "modalities": request_obj.modalities or ["image"],
         }
 
 

python/sglang/srt/managers/scheduler.py

@@ -15,7 +15,6 @@ limitations under the License.
 
 """A scheduler that manages a tensor parallel GPU worker."""
 
-import json
 import logging
 import os
 import threading
@@ -23,7 +22,7 @@ import time
 import warnings
 from collections import deque
 from types import SimpleNamespace
-from typing import List, Optional, Union
+from typing import List, Optional
 
 import torch
 import zmq
@@ -68,8 +67,6 @@ from sglang.srt.utils import (
     broadcast_pyobj,
     configure_logger,
     get_zmq_socket,
-    is_generation_model,
-    is_multimodal_model,
     kill_parent_process,
     set_random_seed,
     suppress_other_loggers,
@@ -133,15 +130,17 @@ class Scheduler:
         # Init tokenizer
         self.model_config = ModelConfig(
             server_args.model_path,
-            server_args.trust_remote_code,
+            trust_remote_code=server_args.trust_remote_code,
             context_length=server_args.context_length,
-            model_override_args=json.loads(server_args.json_model_override_args),
+            model_override_args=server_args.json_model_override_args,
+            is_embedding=server_args.is_embedding,
         )
+        self.is_generation = self.model_config.is_generation
 
         if server_args.skip_tokenizer_init:
             self.tokenizer = self.processor = None
         else:
-            if is_multimodal_model(self.model_config.hf_config.architectures):
+            if self.model_config.is_multimodal:
                 self.processor = get_processor(
                     server_args.tokenizer_path,
                     tokenizer_mode=server_args.tokenizer_mode,
@@ -154,9 +153,6 @@ class Scheduler:
                     tokenizer_mode=server_args.tokenizer_mode,
                     trust_remote_code=server_args.trust_remote_code,
                 )
-        self.is_generation = is_generation_model(
-            self.model_config.hf_config.architectures, self.server_args.is_embedding
-        )
 
         # Launch a tensor parallel worker
         if self.enable_overlap:

python/sglang/srt/managers/tokenizer_manager.py

@@ -18,7 +18,6 @@ limitations under the License.
 import asyncio
 import copy
 import dataclasses
-import json
 import logging
 import os
 import signal
@@ -31,12 +30,8 @@ import zmq
 import zmq.asyncio
 from fastapi import BackgroundTasks
 
-from sglang.srt.hf_transformers_utils import (
-    get_config,
-    get_context_length,
-    get_processor,
-    get_tokenizer,
-)
+from sglang.srt.configs.model_config import ModelConfig
+from sglang.srt.hf_transformers_utils import get_processor, get_tokenizer
 from sglang.srt.managers.image_processor import (
     get_dummy_image_processor,
     get_image_processor,
@@ -59,12 +54,7 @@ from sglang.srt.managers.io_struct import (
 )
 from sglang.srt.sampling.sampling_params import SamplingParams
 from sglang.srt.server_args import PortArgs, ServerArgs
-from sglang.srt.utils import (
-    get_zmq_socket,
-    is_generation_model,
-    is_multimodal_model,
-    kill_child_process,
-)
+from sglang.srt.utils import get_zmq_socket, kill_child_process
 
 asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
 
@@ -103,18 +93,17 @@ class TokenizerManager:
         # Read model args
         self.model_path = server_args.model_path
         self.served_model_name = server_args.served_model_name
-        self.hf_config = get_config(
-            self.model_path,
+        self.model_config = ModelConfig(
+            server_args.model_path,
             trust_remote_code=server_args.trust_remote_code,
-            model_override_args=json.loads(server_args.json_model_override_args),
-        )
-        self.is_generation = is_generation_model(
-            self.hf_config.architectures, self.server_args.is_embedding
-        )
-        self.context_len = server_args.context_length or get_context_length(
-            self.hf_config
+            context_length=server_args.context_length,
+            model_override_args=server_args.json_model_override_args,
+            is_embedding=server_args.is_embedding,
         )
 
+        self.is_generation = self.model_config.is_generation
+        self.context_len = self.model_config.context_len
+
         # Create image processor placeholder
         self.image_processor = get_dummy_image_processor()
 
@@ -122,7 +111,7 @@ class TokenizerManager:
         if server_args.skip_tokenizer_init:
             self.tokenizer = self.processor = None
         else:
-            if is_multimodal_model(self.hf_config.architectures):
+            if self.model_config.is_multimodal:
                 self.processor = get_processor(
                     server_args.tokenizer_path,
                     tokenizer_mode=server_args.tokenizer_mode,
@@ -133,7 +122,7 @@ class TokenizerManager:
 
                 # We want to parallelize the image pre-processing so we create an executor for it
                 self.image_processor = get_image_processor(
-                    self.hf_config, server_args, self.processor
+                    self.model_config.hf_config, server_args, self.processor
                 )
             else:
                 self.tokenizer = get_tokenizer(

python/sglang/srt/managers/tp_worker.py

@@ -15,7 +15,6 @@ limitations under the License.
 
 """A tensor parallel worker."""
 
-import json
 import logging
 from typing import Optional
 
@@ -26,7 +25,7 @@ from sglang.srt.managers.schedule_batch import ModelWorkerBatch, global_server_a
 from sglang.srt.model_executor.forward_batch_info import ForwardBatch
 from sglang.srt.model_executor.model_runner import ModelRunner
 from sglang.srt.server_args import ServerArgs
-from sglang.srt.utils import broadcast_pyobj, is_multimodal_model, set_random_seed
+from sglang.srt.utils import broadcast_pyobj, set_random_seed
 
 logger = logging.getLogger(__name__)
 
@@ -48,9 +47,10 @@ class TpModelWorker:
         # Init model and tokenizer
         self.model_config = ModelConfig(
             server_args.model_path,
-            server_args.trust_remote_code,
+            trust_remote_code=server_args.trust_remote_code,
             context_length=server_args.context_length,
-            model_override_args=json.loads(server_args.json_model_override_args),
+            model_override_args=server_args.json_model_override_args,
+            is_embedding=server_args.is_embedding,
         )
         self.model_runner = ModelRunner(
             model_config=self.model_config,
@@ -64,7 +64,7 @@ class TpModelWorker:
         if server_args.skip_tokenizer_init:
             self.tokenizer = self.processor = None
         else:
-            if is_multimodal_model(self.model_config.hf_config.architectures):
+            if self.model_config.is_multimodal:
                 self.processor = get_processor(
                     server_args.tokenizer_path,
                     tokenizer_mode=server_args.tokenizer_mode,