chenchenghao/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

fix: handle Jinja2 template errors as client errors in OpenAIServingChat (#14748)

Browse Source 
Signed-off-by: Xinyuan Tong <xinyuantong.cs@gmail.com>
This commit is contained in:
Xinyuan Tong
2025-12-12 01:06:26 +00:00
committed by GitHub
parent f832994c32
commit 4885f8b9c1
1 changed files with 22 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

38
python/sglang/srt/entrypoints/openai/serving_chat.py
View File

@@ -7,6 +7,7 @@ import time
import uuid
from typing import TYPE_CHECKING, Any, AsyncGenerator, Dict, List, Optional, Union
import jinja2
import orjson
from fastapi import Request
from fastapi.responses import ORJSONResponse, StreamingResponse
@@ -362,27 +363,32 @@ class OpenAIServingChat(OpenAIServingBase):
else {}
),
)
except Exception:
# This except branch will be triggered when the chosen model
# has a different tools input format that is not compatible
# with openAI's apply_chat_template tool_call format, like Mistral.
except Exception as e:
# If the first attempt fails, try transforming the tools format
# This handles models like Mistral that have a different tools input format
# that is not compatible with OpenAI's apply_chat_template tool_call format
tools = (
[t if "function" in t else {"function": t} for t in tools]
if tools
else None
)
prompt_ids = self.tokenizer_manager.tokenizer.apply_chat_template(
openai_compatible_messages,
tokenize=True,
add_generation_prompt=True,
tools=tools,
reasoning_effort=request.reasoning_effort,
**(
request.chat_template_kwargs
if request.chat_template_kwargs
else {}
),
)
try:
prompt_ids = self.tokenizer_manager.tokenizer.apply_chat_template(
openai_compatible_messages,
tokenize=True,
add_generation_prompt=True,
tools=tools,
reasoning_effort=request.reasoning_effort,
**(
request.chat_template_kwargs
if request.chat_template_kwargs
else {}
),
)
except jinja2.TemplateError as template_error:
# Template errors (e.g., from raise_exception in Jinja templates)
# should be treated as client errors (400 BadRequest)
raise ValueError(str(template_error)) from template_error
if assistant_prefix:
encoded = self.tokenizer_manager.tokenizer.encode(assistant_prefix)