move more files under srt/utils (#11285)

2025-10-09 16:46:15 -07:00
parent 758b887ad1
commit 9b8ebb2798
28 changed files with 96 additions and 55 deletions
--- a/python/sglang/srt/tokenizer/tiktoken_tokenizer.py
+++ b/python/sglang/srt/tokenizer/tiktoken_tokenizer.py
@@ -133,9 +133,9 @@ class TiktokenTokenizer:
        )
        return self.encode(ret) if tokenize else ret

-    def __call__(self, text, **kwargs):
+    def __call__(self, text: List[str], **kwargs):
        return {
-            "input_ids": self.encode(text),
+            "input_ids": [self.encode(x) for x in text],
        }

    def init_xgrammar(self):