Add skip_tokenizer_init args. (#959)

Co-authored-by: lzhang <zhanglei@modelbest.cn>
2024-08-09 12:14:13 -07:00
parent b16e856f11
commit 9cf0a5bada
10 changed files with 218 additions and 71 deletions
@@ -20,10 +20,20 @@ from sglang.srt.constrained.base_tool_cache import BaseToolCache


 class FSMCache(BaseToolCache):
-    def __init__(self, tokenizer_path, tokenizer_args_dict, enable=True):
+    def __init__(
+        self,
+        tokenizer_path,
+        tokenizer_args_dict,
+        enable=True,
+        skip_tokenizer_init=False,
+    ):
        super().__init__(enable=enable)

-        if tokenizer_path.endswith(".json") or tokenizer_path.endswith(".model"):
+        if (
+            skip_tokenizer_init
+            or tokenizer_path.endswith(".json")
+            or tokenizer_path.endswith(".model")
+        ):
            # Do not support TiktokenTokenizer or SentencePieceTokenizer
            return