chenchenghao/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 11 Packages Projects Releases Wiki Activity

[GLM4.6v] Required changes for bumping up to transformer 5.x (#13229)

Browse Source
This commit is contained in:
Binyao Jiang
2025-11-17 18:58:00 -08:00
committed by GitHub
parent d7984f3125
commit 90c18a16cb
11 changed files with 29 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
docs/advanced_features/separate_reasoning.ipynb
View File

@@ -256,9 +256,7 @@
"\n",
"tokenizer = AutoTokenizer.from_pretrained(\"deepseek-ai/DeepSeek-R1-Distill-Qwen-7B\")\n",
"input = tokenizer.apply_chat_template(\n",
" messages,\n",
" tokenize=False,\n",
" add_generation_prompt=True,\n",
" messages, tokenize=False, add_generation_prompt=True, return_dict=False\n",
")\n",
"\n",
"gen_url = f\"http://localhost:{port}/generate\"\n",
@@ -319,9 +317,7 @@
"llm = sgl.Engine(model_path=\"deepseek-ai/DeepSeek-R1-Distill-Qwen-7B\")\n",
"tokenizer = AutoTokenizer.from_pretrained(\"deepseek-ai/DeepSeek-R1-Distill-Qwen-7B\")\n",
"input = tokenizer.apply_chat_template(\n",
" messages,\n",
" tokenize=False,\n",
" add_generation_prompt=True,\n",
" messages, tokenize=False, add_generation_prompt=True, return_dict=False\n",
")\n",
"sampling_params = {\n",
" \"max_new_tokens\": 1024,\n",

10
docs/advanced_features/structured_outputs.ipynb
View File

@@ -443,7 +443,7 @@
" }\n",
"]\n",
"text = tokenizer.apply_chat_template(\n",
" messages, tokenize=False, add_generation_prompt=True\n",
" messages, tokenize=False, add_generation_prompt=True, return_dict=False\n",
")\n",
"response = requests.post(\n",
" f\"http://localhost:{port}/generate\",\n",
@@ -525,7 +525,7 @@
" }\n",
"]\n",
"text = tokenizer.apply_chat_template(\n",
" messages, tokenize=False, add_generation_prompt=True\n",
" messages, tokenize=False, add_generation_prompt=True, return_dict=False\n",
")\n",
"response = requests.post(\n",
" f\"http://localhost:{port}/generate\",\n",
@@ -571,7 +571,7 @@
" }\n",
"]\n",
"text = tokenizer.apply_chat_template(\n",
" messages, tokenize=False, add_generation_prompt=True\n",
" messages, tokenize=False, add_generation_prompt=True, return_dict=False\n",
")\n",
"response = requests.post(\n",
" f\"http://localhost:{port}/generate\",\n",
@@ -606,7 +606,7 @@
"tokenizer = AutoTokenizer.from_pretrained(\"meta-llama/Meta-Llama-3.1-8B-Instruct\")\n",
"\n",
"text = tokenizer.apply_chat_template(\n",
" messages, tokenize=False, add_generation_prompt=True\n",
" messages, tokenize=False, add_generation_prompt=True, return_dict=False\n",
")\n",
"payload = {\n",
" \"text\": text,\n",
@@ -883,7 +883,7 @@
"outputs": [],
"source": [
"text = tokenizer.apply_chat_template(\n",
" messages, tokenize=False, add_generation_prompt=True\n",
" messages, tokenize=False, add_generation_prompt=True, return_dict=False\n",
")\n",
"prompts = [text]\n",
"\n",

8
docs/advanced_features/structured_outputs_for_reasoning_models.ipynb
View File

@@ -400,7 +400,7 @@
" },\n",
"]\n",
"text = tokenizer.apply_chat_template(\n",
" messages, tokenize=False, add_generation_prompt=True\n",
" messages, tokenize=False, add_generation_prompt=True, return_dict=False\n",
")\n",
"# Make API request\n",
"response = requests.post(\n",
@@ -448,7 +448,7 @@
"\n",
"# JSON\n",
"text = tokenizer.apply_chat_template(\n",
" messages, tokenize=False, add_generation_prompt=True\n",
" messages, tokenize=False, add_generation_prompt=True, return_dict=False\n",
")\n",
"response = requests.post(\n",
" f\"http://localhost:{port}/generate\",\n",
@@ -543,7 +543,7 @@
"outputs": [],
"source": [
"text = tokenizer.apply_chat_template(\n",
" messages, tokenize=False, add_generation_prompt=True\n",
" messages, tokenize=False, add_generation_prompt=True, return_dict=False\n",
")\n",
"payload = {\n",
" \"text\": text,\n",
@@ -765,7 +765,7 @@
"outputs": [],
"source": [
"text = tokenizer.apply_chat_template(\n",
" messages, tokenize=False, add_generation_prompt=True\n",
" messages, tokenize=False, add_generation_prompt=True, return_dict=False\n",
")\n",
"prompts = [text]\n",
"\n",

7
docs/advanced_features/tool_parser.ipynb
View File

@@ -391,10 +391,7 @@
"messages = get_messages()\n",
"\n",
"input = tokenizer.apply_chat_template(\n",
" messages,\n",
" tokenize=False,\n",
" add_generation_prompt=True,\n",
" tools=tools,\n",
" messages, tokenize=False, add_generation_prompt=True, tools=tools, return_dict=False\n",
")\n",
"\n",
"gen_url = f\"http://localhost:{port}/generate\"\n",
@@ -459,7 +456,7 @@
"llm = sgl.Engine(model_path=\"Qwen/Qwen2.5-7B-Instruct\")\n",
"tokenizer = llm.tokenizer_manager.tokenizer\n",
"input_ids = tokenizer.apply_chat_template(\n",
" messages, tokenize=True, add_generation_prompt=True, tools=tools\n",
" messages, tokenize=True, add_generation_prompt=True, tools=tools, return_dict=False\n",
")\n",
"\n",
"# Note that for gpt-oss tool parser, adding \"no_stop_trim\": True\n",

2
docs/basic_usage/native_api.ipynb
View File

@@ -410,7 +410,7 @@
"]\n",
"\n",
"tokenizer = AutoTokenizer.from_pretrained(\"Skywork/Skywork-Reward-Llama-3.1-8B-v0.2\")\n",
"prompts = tokenizer.apply_chat_template(CONVS, tokenize=False)\n",
"prompts = tokenizer.apply_chat_template(CONVS, tokenize=False, return_dict=False)\n",
"\n",
"url = f\"http://localhost:{port}/classify\"\n",
"data = {\"model\": \"Skywork/Skywork-Reward-Llama-3.1-8B-v0.2\", \"text\": prompts}\n",

6
python/sglang/bench_serving.py
View File

@@ -995,7 +995,10 @@ async def get_mooncake_request_over_time(
# Form the full prompt from history
try:
full_prompt_text = tokenizer.apply_chat_template(
chat_history, tokenize=False, add_generation_prompt=True
chat_history,
tokenize=False,
add_generation_prompt=True,
return_dict=False,
)
except Exception:
full_prompt_text = "\n".join(
@@ -1161,6 +1164,7 @@ def sample_sharegpt_requests(
[{"role": "user", "content": prompt}],
add_generation_prompt=True,
tokenize=False,
return_dict=False,
)
if tokenizer.bos_token:
prompt = prompt.replace(tokenizer.bos_token, "")

3
python/sglang/multimodal_gen/runtime/loader/weight_utils.py
View File

@@ -46,7 +46,8 @@ enable_hf_transfer()
class DisabledTqdm(tqdm):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs, disable=True)
kwargs["disable"] = True
super().__init__(*args, **kwargs)
def get_lock(model_name_or_path: str | Path, cache_dir: str | None = None):

2
python/sglang/srt/entrypoints/openai/serving_chat.py
View File

@@ -324,6 +324,7 @@ class OpenAIServingChat(OpenAIServingBase):
**(
request.chat_template_kwargs if request.chat_template_kwargs else {}
),
return_dict=False,
)
except Exception:
# This except branch will be triggered when the chosen model
@@ -343,6 +344,7 @@ class OpenAIServingChat(OpenAIServingBase):
**(
request.chat_template_kwargs if request.chat_template_kwargs else {}
),
return_dict=False,
)
if assistant_prefix:

3
python/sglang/srt/model_loader/weight_utils.py
View File

@@ -70,7 +70,8 @@ enable_hf_transfer()
class DisabledTqdm(tqdm):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs, disable=True)
kwargs["disable"] = True
super().__init__(*args, **kwargs)
def get_lock(model_name_or_path: str, cache_dir: Optional[str] = None):

2
python/sglang/test/runners.py
View File

@@ -353,7 +353,7 @@ class HFRunner:
scores = []
for conv in prompts:
conv_formatted = self.tokenizer.apply_chat_template(
conv, tokenize=False
conv, tokenize=False, return_dict=False
)
conv_tokenized = self.tokenizer(
conv_formatted, return_tensors="pt"

4
test/srt/models/test_reward_models.py
View File

@@ -68,7 +68,9 @@ class TestRewardModels(CustomTestCase):
torch_dtype=torch_dtype,
model_type="reward",
) as srt_runner:
prompts = srt_runner.tokenizer.apply_chat_template(convs, tokenize=False)
prompts = srt_runner.tokenizer.apply_chat_template(
convs, tokenize=False, return_dict=False
)
srt_outputs = srt_runner.forward(prompts)
hf_scores = torch.tensor(hf_outputs.scores)