Higher priority for user input of max_prefill_tokens & format (#540)

python/sglang/srt/model_config.py

@@ -1,8 +1,9 @@
 from typing import Optional
 
-from sglang.srt.hf_transformers_utils import get_config, get_context_length
 from transformers import PretrainedConfig
 
+from sglang.srt.hf_transformers_utils import get_config, get_context_length
+
 
 class ModelConfig:
     def __init__(
@@ -17,8 +18,12 @@ class ModelConfig:
         self.trust_remote_code = trust_remote_code
         self.revision = revision
         self.model_overide_args = model_overide_args
-        self.hf_config = get_config(self.path, trust_remote_code, revision,
-                                    model_overide_args=model_overide_args)
+        self.hf_config = get_config(
+            self.path,
+            trust_remote_code,
+            revision,
+            model_overide_args=model_overide_args,
+        )
         self.hf_text_config = get_hf_text_config(self.hf_config)
         if context_length is not None:
             self.context_len = context_length
@@ -55,18 +60,23 @@ class ModelConfig:
         # KV heads.
         falcon_model_types = ["falcon", "RefinedWeb", "RefinedWebModel"]
         new_decoder_arch_falcon = (
-                self.hf_config.model_type in falcon_model_types
-                and getattr(self.hf_config, "new_decoder_architecture", False))
-        if not new_decoder_arch_falcon and getattr(self.hf_text_config,
-                                                   "multi_query", False):
+            self.hf_config.model_type in falcon_model_types
+            and getattr(self.hf_config, "new_decoder_architecture", False)
+        )
+        if not new_decoder_arch_falcon and getattr(
+            self.hf_text_config, "multi_query", False
+        ):
             # Multi-query attention, only one KV head.
             # Currently, tensor parallelism is not supported in this case.
             return 1
 
         # For DBRX and MPT
         if self.hf_config.model_type in ["dbrx", "mpt"]:
-            return getattr(self.hf_config.attn_config, "kv_n_heads",
-                           self.hf_config.num_attention_heads)
+            return getattr(
+                self.hf_config.attn_config,
+                "kv_n_heads",
+                self.hf_config.num_attention_heads,
+            )
 
         attributes = [
             # For Falcon:
@@ -94,13 +104,12 @@ class ModelConfig:
         # the tensor parallel size. We will replicate the KV heads in the
         # case where the number of KV heads is smaller than the tensor
         # parallel size so each GPU has at least one KV head.
-        return max(1,
-                   total_num_kv_heads // tensor_parallel_size)
+        return max(1, total_num_kv_heads // tensor_parallel_size)
 
 
 def get_hf_text_config(config: PretrainedConfig):
     """Get the "sub" config relevant to llm for multi modal models.
-        No op for pure text models.
+    No op for pure text models.
     """
     if hasattr(config, "text_config"):
         # The code operates under the assumption that text_config should have