From 7c38eca1e4a704bf09fe6b52ea040a41d3cfc55d Mon Sep 17 00:00:00 2001 From: Eva20150932-atlascloud Date: Tue, 2 Dec 2025 19:41:05 +0000 Subject: [PATCH] feat: DeepSeek new v3.2 encoding (#14249) Co-authored-by: Xinyuan Tong --- .../srt/entrypoints/openai/encoding_dsv32.py | 451 ++++++++++++++++++ .../srt/entrypoints/openai/serving_chat.py | 201 ++++---- .../srt/function_call/deepseekv32_detector.py | 321 +++++++++++++ .../srt/function_call/function_call_parser.py | 2 + .../test_function_call_parser.py | 201 ++++++++ .../openai_server/basic/test_serving_chat.py | 72 ++- 6 files changed, 1156 insertions(+), 92 deletions(-) create mode 100644 python/sglang/srt/entrypoints/openai/encoding_dsv32.py create mode 100644 python/sglang/srt/function_call/deepseekv32_detector.py diff --git a/python/sglang/srt/entrypoints/openai/encoding_dsv32.py b/python/sglang/srt/entrypoints/openai/encoding_dsv32.py new file mode 100644 index 000000000..7d314d169 --- /dev/null +++ b/python/sglang/srt/entrypoints/openai/encoding_dsv32.py @@ -0,0 +1,451 @@ +# Adapted from https://huggingface.co/deepseek-ai/DeepSeek-V3.2/blob/main/encoding/encoding_dsv32.py +import copy +import json +import re +from typing import Any, Dict, List, Optional, Tuple, Union + +TOOLS_SYSTEM_TEMPLATE = """## Tools +You have access to a set of tools you can use to answer the user's question. +You can invoke functions by writing a "<{dsml_token}function_calls>" block like the following as part of your reply to the user: +<{dsml_token}function_calls> +<{dsml_token}invoke name="$FUNCTION_NAME"> +<{dsml_token}parameter name="$PARAMETER_NAME" string="true|false">$PARAMETER_VALUE +... + +<{dsml_token}invoke name="$FUNCTION_NAME2"> +... + + +String and scalar parameters should be specified as is without any escaping or quotes, while lists and objects should use JSON format. The "string" attribute should be set to "true" for string type parameters and "false" for other types (numbers, booleans, arrays, objects). +If the thinking_mode is enabled, then after function results you should strongly consider outputting a thinking block. Here is an example: +<{dsml_token}function_calls> +... + + +... + +{thinking_start_token}...thinking about results{thinking_end_token} +Here are the functions available in JSONSchema format: + +{tool_schemas} + +""" + +bos_token: str = "<｜begin▁of▁sentence｜>" +eos_token: str = "<｜end▁of▁sentence｜>" +thinking_start_token: str = "" +thinking_end_token: str = "" +dsml_token: str = "｜DSML｜" +system_msg_template: str = "{content}" +user_msg_template: str = "<｜User｜>{content}<｜Assistant｜>" +assistant_msg_template: str = "{reasoning}{content}{tool_calls}<｜end▁of▁sentence｜>" +thinking_template = "{reasoning_content}" + +response_format_template: str = ( + "## Response Format:\n\nYou MUST strictly adhere to the following schema to reply:\n{schema}" +) +tool_call_template: str = ( + '<{dsml_token}invoke name="{name}">\n{arguments}\n' +) +tool_calls_template = ( + "<{dsml_token}function_calls>\n{tool_calls}\n" +) + +tool_output_template: str = "\n{content}" + + +def to_json(value: Any) -> str: + try: + return json.dumps(value, ensure_ascii=False) + except: + return json.dumps(value, ensure_ascii=True) + + +def tools_from_openai_format(tools): + return [tool["function"] for tool in tools] + + +def tool_calls_from_openai_format(tool_calls): + return [ + { + "name": tool_call["function"]["name"], + "arguments": tool_call["function"]["arguments"], + } + for tool_call in tool_calls + ] + + +def tool_calls_to_openai_format(tool_calls): + return [ + { + "type": "function", + "function": { + "name": tool_call["name"], + "arguments": tool_call["arguments"], + }, + } + for tool_call in tool_calls + ] + + +def encode_arguments_to_dsml(tool_call: Dict[str, str]) -> str: + p_dsml_template = """<{dsml_token}parameter name="{key}" string="{is_str}">{value}""" + P_dsml_strs = [] + + arguments = json.loads(tool_call["arguments"]) + + for k, v in arguments.items(): + p_dsml_str = p_dsml_template.format( + dsml_token=dsml_token, + key=k, + is_str="true" if isinstance(v, str) else "false", + value=v if isinstance(v, str) else to_json(v), + ) + + P_dsml_strs.append(p_dsml_str) + + return "\n".join(P_dsml_strs) + + +def decode_dsml_to_arguments( + tool_name: str, tool_args: Dict[str, Tuple[str, str]] +) -> Dict[str, str]: + def _decode_value(key: str, value: str, string: str): + if string == "true": + value = to_json(value) + return f"{to_json(key)}: {value}" + + tool_args_json = ( + "{" + + ", ".join( + [_decode_value(k, v, string=is_str) for k, (v, is_str) in tool_args.items()] + ) + + "}" + ) + return dict(name=tool_name, arguments=tool_args_json) + + +def render_tools(tools: List[Dict[str, Union[str, Dict[str, Any]]]]) -> str: + tools_json = [to_json(t) for t in tools] + + return TOOLS_SYSTEM_TEMPLATE.format( + tool_schemas="\n".join(tools_json), + dsml_token=dsml_token, + thinking_start_token=thinking_start_token, + thinking_end_token=thinking_end_token, + ) + + +def find_last_user_index(messages: List[Dict[str, Any]]) -> int: + last_user_index = -1 + for idx in range(len(messages) - 1, -1, -1): + if messages[idx].get("role") in ["user", "developer"]: + last_user_index = idx + break + return last_user_index + + +def render_message( + index: int, messages: List[Dict[str, Any]], thinking_mode: str +) -> str: + assert 0 <= index < len(messages) + assert thinking_mode in [ + "chat", + "thinking", + ], f"Invalid thinking_mode `{thinking_mode}`" + + prompt = "" + msg = messages[index] + last_user_idx = find_last_user_index(messages) + + role = msg.get("role") + content = msg.get("content") + tools = msg.get("tools") + response_format = msg.get("response_format") + tool_calls = msg.get("tool_calls") + reasoning_content = msg.get("reasoning_content") + + if tools: + tools = tools_from_openai_format(tools) + if tool_calls: + tool_calls = tool_calls_from_openai_format(tool_calls) + + if role == "system": + prompt += system_msg_template.format(content=content or "") + if tools: + prompt += "\n\n" + render_tools(tools) + + if response_format: + prompt += "\n\n" + response_format_template.format( + schema=to_json(response_format) + ) + + elif role == "developer": + assert content, f"Invalid message for role `{role}`: {msg}" + content_developer = "" + if tools: + content_developer += "\n\n" + render_tools(tools) + + if response_format: + content_developer += "\n\n" + response_format_template.format( + schema=to_json(response_format) + ) + + content_developer += "\n\n# The user's message is: {}".format(content) + + prompt += user_msg_template.format(content=content_developer) + if index == last_user_idx and thinking_mode == "thinking": + prompt += thinking_start_token + else: + prompt += thinking_end_token + + elif role == "user": + prompt += user_msg_template.format(content=content) + + if index == last_user_idx and thinking_mode == "thinking": + prompt += thinking_start_token + else: + prompt += thinking_end_token + + elif role == "tool": + prev_assistant_idx = index - 1 + assistant_msg = messages[prev_assistant_idx] + while prev_assistant_idx >= 0 and assistant_msg.get("role") == "tool": + prev_assistant_idx -= 1 + assistant_msg = messages[prev_assistant_idx] + + assert ( + index == 0 + or prev_assistant_idx >= 0 + and assistant_msg.get("role") == "assistant" + ), f"Invalid messages at {index}:\n{assistant_msg}" + + tool_call_order = index - prev_assistant_idx + assistant_tool_calls = assistant_msg.get("tool_calls") + assert ( + assistant_tool_calls and len(assistant_tool_calls) >= tool_call_order + ), "No tool calls but found tool output" + + if tool_call_order == 1: + prompt += "\n\n" + + prompt += tool_output_template.format(content=content) + + if tool_call_order == len(assistant_tool_calls): + prompt += "\n" + + if index >= last_user_idx and thinking_mode == "thinking": + prompt += "\n\n" + thinking_start_token + else: + prompt += "\n\n" + thinking_end_token + + elif role == "assistant": + prev_assistant_idx = index + thinking_part = "" + + tool_calls_content = "" + if tool_calls: + tool_calls = [ + tool_call_template.format( + dsml_token=dsml_token, + name=tool_call.get("name"), + arguments=encode_arguments_to_dsml(tool_call), + ) + for tool_call in tool_calls + ] + tool_calls_content += "\n\n" + tool_calls_template.format( + dsml_token=dsml_token, tool_calls="\n".join(tool_calls) + ) + + summary_content = content or "" + + if thinking_mode == "thinking" and index > last_user_idx: + assert ( + reasoning_content or tool_calls + ), f"ThinkingMode: {thinking_mode}, invalid message without reasoning_content/tool_calls `{msg}` after last user message" + thinking_part = ( + thinking_template.format(reasoning_content=reasoning_content or "") + + thinking_end_token + ) + + prompt += assistant_msg_template.format( + reasoning=thinking_part, + content=summary_content, + tool_calls=tool_calls_content, + ) + else: + raise NotImplementedError(f"Unknown role: {role}") + + return prompt + + +def drop_thinking_messages( + messages: List[Dict[str, Any]], last_user_idx: Optional[int] = None +) -> List[Dict[str, Any]]: + messages_wo_thinking: List[Dict[str, Any]] = [] + last_user_idx = ( + find_last_user_index(messages) if last_user_idx is None else last_user_idx + ) + for idx, msg in enumerate(messages): + role = msg.get("role") + if role in ["user", "system", "tool"] or idx >= last_user_idx: + messages_wo_thinking.append(msg) + continue + + elif role == "assistant": + msg_wo_thinking = copy.copy(msg) + msg_wo_thinking.pop("reasoning_content", None) + messages_wo_thinking.append(msg_wo_thinking) + + return messages_wo_thinking + + +def encode_messages( + messages: List[Dict[str, Any]], + thinking_mode: str, + context: Optional[List[Dict[str, Any]]] = None, + drop_thinking: bool = True, + add_default_bos_token: bool = True, +) -> str: + context = context if context else [] + full_messages = context + messages + + prompt = bos_token if add_default_bos_token and len(context) == 0 else "" + + if thinking_mode == "thinking" and drop_thinking: + full_messages = drop_thinking_messages(full_messages) + + for idx in range(len(messages)): + prompt += render_message( + idx + len(context), full_messages, thinking_mode=thinking_mode + ) + + return prompt + + +def _read_until_stop( + index: int, text: str, stop: List[str] +) -> Tuple[int, str, Optional[str]]: + min_pos = len(text) + matched_stop = None + + for s in stop: + pos = text.find(s, index) + if pos != -1 and pos < min_pos: + min_pos = pos + matched_stop = s + + if matched_stop: + content = text[index:min_pos] + return min_pos + len(matched_stop), content, matched_stop + else: + content = text[index:] + return len(text), content, None + + +def parse_tool_calls(index: int, text: str): + tool_calls: List[Dict[str, Any]] = [] + stop_token = None + tool_calls_end_token = f"" + + while index < len(text): + index, _, stop_token = _read_until_stop( + index, text, [f"<{dsml_token}invoke", tool_calls_end_token] + ) + assert _ == ">\n", "Tool call format error" + + if stop_token == tool_calls_end_token: + break + + assert stop_token is not None, "Missing special token" + + index, tool_name_content, stop_token = _read_until_stop( + index, text, [f"<{dsml_token}parameter", f"\n$', tool_name_content, flags=re.DOTALL + ) + assert len(p_tool_name) == 1, "Tool name format error" + tool_name = p_tool_name[0] + + tool_args: Dict[str, Tuple[str, str]] = {} + while stop_token == f"<{dsml_token}parameter": + index, param_content, stop_token = _read_until_stop( + index, text, [f"/{dsml_token}parameter"] + ) + + param_kv = re.findall( + r'^ name="(.*?)" string="(true|false)">(.*?)<$', + param_content, + flags=re.DOTALL, + ) + assert len(param_kv) == 1, "Parameter format error" + param_name, string, param_value = param_kv[0] + + assert param_name not in tool_args, "Duplicate parameter name" + tool_args[param_name] = (param_value, string) + + index, content, stop_token = _read_until_stop( + index, text, [f"<{dsml_token}parameter", f"\n", "Parameter format error" + + tool_call = decode_dsml_to_arguments(tool_name=tool_name, tool_args=tool_args) + tool_calls.append(tool_call) + + return index, stop_token, tool_calls + + +# NOTE: This function is designed to parse only correctly formatted string and will not attempt to correct malformed output that may be generated by the model. +def parse_message_from_completion_text(text: str, thinking_mode: str): + summary_content, reasoning_content, tool_calls = "", "", [] + index, stop_token = 0, None + tool_calls_start_token = f"\n\n<{dsml_token}function_calls" + + is_thinking, is_tool_calling = thinking_mode == "thinking", False + + if is_thinking: + index, content_delta, stop_token = _read_until_stop( + index, text, [thinking_end_token, tool_calls_start_token] + ) + reasoning_content = content_delta + assert stop_token == thinking_end_token, "Invalid thinking format" + + index, content_delta, stop_token = _read_until_stop( + index, text, [eos_token, tool_calls_start_token] + ) + summary_content = content_delta + if stop_token == tool_calls_start_token: + is_tool_calling = True + else: + assert stop_token == eos_token, "Invalid summary format" + + if is_tool_calling: + index, stop_token, tool_calls = parse_tool_calls(index, text) + + index, tool_ends_text, stop_token = _read_until_stop(index, text, [eos_token]) + assert not tool_ends_text, "Unexpected content after tool calls" + + assert len(text) == index and stop_token in [ + eos_token, + None, + ], "Unexpected content at end" + + for sp_token in [ + bos_token, + eos_token, + thinking_start_token, + thinking_end_token, + dsml_token, + ]: + assert ( + sp_token not in summary_content and sp_token not in reasoning_content + ), "Unexpected special token in content" + + return { + "role": "assistant", + "content": summary_content, + "reasoning_content": reasoning_content, + "tool_calls": tool_calls_to_openai_format(tool_calls), + } diff --git a/python/sglang/srt/entrypoints/openai/serving_chat.py b/python/sglang/srt/entrypoints/openai/serving_chat.py index 77903c5bb..48197bd78 100644 --- a/python/sglang/srt/entrypoints/openai/serving_chat.py +++ b/python/sglang/srt/entrypoints/openai/serving_chat.py @@ -12,6 +12,7 @@ from fastapi import Request from fastapi.responses import ORJSONResponse, StreamingResponse from jsonschema import Draft202012Validator, SchemaError +from sglang.srt.entrypoints.openai.encoding_dsv32 import encode_messages from sglang.srt.entrypoints.openai.protocol import ( ChatCompletionRequest, ChatCompletionResponse, @@ -82,6 +83,17 @@ class OpenAIServingChat(OpenAIServingBase): and self.tokenizer_manager.model_config.hf_config.model_type == "gpt_oss" ) + self.use_dpsk_v32_encoding = self._use_dpsk_v32_encoding() + + def _use_dpsk_v32_encoding(self) -> bool: + has_chat_template = ( + self.tokenizer_manager.tokenizer is not None + and self.tokenizer_manager.tokenizer.chat_template is not None + ) + architectures = self.tokenizer_manager.server_args.get_hf_config().architectures + is_dpsk_v32 = "DeepseekV3" in architectures[0] if architectures else False + return not has_chat_template and is_dpsk_v32 + def _request_id_prefix(self) -> str: return "chatcmpl-" @@ -270,92 +282,117 @@ class OpenAIServingChat(OpenAIServingBase): template_content_format = self.template_manager.jinja_template_content_format - for message in request.messages: - if message.content is None: - message.content = "" - msg_dict = message.model_dump() - - # Process content based on detected template format - processed_msg = process_content_for_template_format( - msg_dict, - template_content_format, - image_data, - video_data, - audio_data, - modalities, - ) - - # per the Transformers docs & maintainers, tool call arguments in - # assistant-role messages with tool_calls need to be dicts not JSON str - - # this is how tool-use chat templates will expect them moving forwards - # so, for messages that have tool_calls, parse the string (which we get - # from openAI format) to dict - if ( - processed_msg["role"] == "assistant" - and "tool_calls" in processed_msg - and isinstance(processed_msg["tool_calls"], list) + if self.use_dpsk_v32_encoding: + if request.chat_template_kwargs and request.chat_template_kwargs.get( + "thinking" ): - for item in processed_msg["tool_calls"]: - if "arguments" in item["function"] and isinstance( - item["function"]["arguments"], str - ): - item["function"]["arguments"] = orjson.loads( - item["function"]["arguments"] - ) - - openai_compatible_messages.append(processed_msg) - - # Handle assistant prefix for continue_final_message - assistant_prefix = None - if ( - openai_compatible_messages - and openai_compatible_messages[-1]["role"] == "assistant" - ): - if request.continue_final_message: - assistant_prefix = openai_compatible_messages[-1]["content"] - openai_compatible_messages = openai_compatible_messages[:-1] - - try: - prompt_ids = self.tokenizer_manager.tokenizer.apply_chat_template( - openai_compatible_messages, - tokenize=True, - add_generation_prompt=True, - tools=tools, - reasoning_effort=request.reasoning_effort, - **( - request.chat_template_kwargs if request.chat_template_kwargs else {} - ), - return_dict=False, - ) - except Exception: - # This except branch will be triggered when the chosen model - # has a different tools input format that is not compatible - # with openAI's apply_chat_template tool_call format, like Mistral. - tools = ( - [t if "function" in t else {"function": t} for t in tools] - if tools - else None - ) - prompt_ids = self.tokenizer_manager.tokenizer.apply_chat_template( - openai_compatible_messages, - tokenize=True, - add_generation_prompt=True, - tools=tools, - reasoning_effort=request.reasoning_effort, - **( - request.chat_template_kwargs if request.chat_template_kwargs else {} - ), - return_dict=False, + thinking_mode = "thinking" + else: + thinking_mode = "chat" + messages = request.messages + messages = [msg.model_dump() for msg in messages] + if messages[0]["role"] != "system": + messages.insert( + 0, {"role": "system", "content": "You are a helpful Assistant."} + ) + if request.tools: + messages[0]["tools"] = [tool.model_dump() for tool in request.tools] + real_input = encode_messages( + messages, thinking_mode=thinking_mode, drop_thinking=False ) + prompt_ids = self.tokenizer_manager.tokenizer.encode(real_input) + else: + for message in request.messages: + if message.content is None: + message.content = "" + msg_dict = message.model_dump() - if assistant_prefix: - encoded = self.tokenizer_manager.tokenizer.encode(assistant_prefix) - if encoded and encoded[0] == self.tokenizer_manager.tokenizer.bos_token_id: - encoded = encoded[1:] - prompt_ids += encoded + # Process content based on detected template format + processed_msg = process_content_for_template_format( + msg_dict, + template_content_format, + image_data, + video_data, + audio_data, + modalities, + ) - if is_multimodal: - prompt = self.tokenizer_manager.tokenizer.decode(prompt_ids) + # per the Transformers docs & maintainers, tool call arguments in + # assistant-role messages with tool_calls need to be dicts not JSON str - + # this is how tool-use chat templates will expect them moving forwards + # so, for messages that have tool_calls, parse the string (which we get + # from openAI format) to dict + if ( + processed_msg["role"] == "assistant" + and "tool_calls" in processed_msg + and isinstance(processed_msg["tool_calls"], list) + ): + for item in processed_msg["tool_calls"]: + if "arguments" in item["function"] and isinstance( + item["function"]["arguments"], str + ): + item["function"]["arguments"] = orjson.loads( + item["function"]["arguments"] + ) + + openai_compatible_messages.append(processed_msg) + + # Handle assistant prefix for continue_final_message + assistant_prefix = None + if ( + openai_compatible_messages + and openai_compatible_messages[-1]["role"] == "assistant" + ): + if request.continue_final_message: + assistant_prefix = openai_compatible_messages[-1]["content"] + openai_compatible_messages = openai_compatible_messages[:-1] + + try: + prompt_ids = self.tokenizer_manager.tokenizer.apply_chat_template( + openai_compatible_messages, + tokenize=True, + add_generation_prompt=True, + tools=tools, + reasoning_effort=request.reasoning_effort, + **( + request.chat_template_kwargs + if request.chat_template_kwargs + else {} + ), + ) + except Exception: + # This except branch will be triggered when the chosen model + # has a different tools input format that is not compatible + # with openAI's apply_chat_template tool_call format, like Mistral. + tools = ( + [t if "function" in t else {"function": t} for t in tools] + if tools + else None + ) + prompt_ids = self.tokenizer_manager.tokenizer.apply_chat_template( + openai_compatible_messages, + tokenize=True, + add_generation_prompt=True, + tools=tools, + reasoning_effort=request.reasoning_effort, + **( + request.chat_template_kwargs + if request.chat_template_kwargs + else {} + ), + ) + + if assistant_prefix: + encoded = self.tokenizer_manager.tokenizer.encode(assistant_prefix) + if ( + encoded + and encoded[0] == self.tokenizer_manager.tokenizer.bos_token_id + ): + encoded = encoded[1:] + prompt_ids += encoded + + if is_multimodal: + prompt = self.tokenizer_manager.tokenizer.decode(prompt_ids) stop = request.stop image_data = image_data if image_data else None diff --git a/python/sglang/srt/function_call/deepseekv32_detector.py b/python/sglang/srt/function_call/deepseekv32_detector.py new file mode 100644 index 000000000..8b28494a5 --- /dev/null +++ b/python/sglang/srt/function_call/deepseekv32_detector.py @@ -0,0 +1,321 @@ +import json +import logging +import re +from typing import List + +from sglang.srt.entrypoints.openai.protocol import Tool +from sglang.srt.function_call.base_format_detector import BaseFormatDetector +from sglang.srt.function_call.core_types import ( + StreamingParseResult, + StructureInfo, + ToolCallItem, + _GetInfoFunc, +) + +logger = logging.getLogger(__name__) + + +class DeepSeekV32Detector(BaseFormatDetector): + """ + Detector for DeepSeek V3.2 model function call format. + + The DeepSeek V3.2 format uses XML-like DSML tags to delimit function calls. + Supports two parameter formats: + + Format 1 - XML Parameter Tags: + ``` + <｜DSML｜function_calls> + <｜DSML｜invoke name="function_name"> + <｜DSML｜parameter name="param_name" string="true">value + ... + + + ``` + + Format 2 - Direct JSON: + ``` + <｜DSML｜function_calls> + <｜DSML｜invoke name="function_name"> + { + "param_name": "value" + } + + + ``` + + Examples: + ``` + <｜DSML｜function_calls> + <｜DSML｜invoke name="get_favorite_tourist_spot"> + <｜DSML｜parameter name="city" string="true">San Francisco + + + + <｜DSML｜function_calls> + <｜DSML｜invoke name="get_favorite_tourist_spot"> + { "city": "San Francisco" } + + + ``` + + Key Components: + - Tool Calls Section: Wrapped between `<｜DSML｜function_calls>` and `` + - Individual Tool Call: Wrapped between `<｜DSML｜invoke name="...">` and `` + - Parameters: Either XML tags or direct JSON format + - Supports multiple tool calls + + Reference: DeepSeek V3.2 format specification + """ + + def __init__(self): + super().__init__() + self.bot_token = "<｜DSML｜function_calls>" + self.eot_token = "" + self.invoke_begin_regex = r'<｜DSML｜invoke\s+name="([^"]+)"\s*>' + self.invoke_end_token = "" + self.parameter_regex = r'<｜DSML｜parameter\s+name="([^"]+)"\s+string="([^"]+)"\s*>(.*?)' + self._last_arguments = "" + self.current_tool_id = -1 + + def has_tool_call(self, text: str) -> bool: + """Check if the text contains a deepseek v32 format tool call.""" + return self.bot_token in text + + def _parse_parameters_from_xml(self, invoke_content: str) -> dict: + """ + Parse parameters from either XML-like format or JSON format to dict. + + Supports two formats: + 1. XML parameter tags: <｜DSML｜parameter name="..." string="...">value + 2. Direct JSON: { "key": "value" } + """ + # First, try to parse as direct JSON (new format) + invoke_content_stripped = invoke_content.strip() + + if invoke_content_stripped.startswith("{") and invoke_content_stripped.endswith( + "}" + ): + try: + parameters = json.loads(invoke_content_stripped) + if isinstance(parameters, dict): + return parameters + except (json.JSONDecodeError, ValueError): + # If JSON parsing fails, fall through to XML parsing + pass + + # Fall back to XML parameter tag parsing (original format) + parameters = {} + param_matches = re.findall(self.parameter_regex, invoke_content, re.DOTALL) + for param_name, param_type, param_value in param_matches: + # Convert value based on type + if param_type == "true": # string type + parameters[param_name] = param_value.strip() + else: + # Try to parse as JSON for other types + try: + parameters[param_name] = json.loads(param_value.strip()) + except (json.JSONDecodeError, ValueError): + parameters[param_name] = param_value.strip() + return parameters + + def detect_and_parse(self, text: str, tools: List[Tool]) -> StreamingParseResult: + """ + One-time parsing: Detects and parses tool calls in the provided text. + + :param text: The complete text to parse. + :param tools: List of available tools. + :return: ParseResult indicating success or failure, consumed text, leftover text, and parsed calls. + """ + idx = text.find(self.bot_token) + normal_text = text[:idx].strip() if idx != -1 else text + if self.bot_token not in text: + return StreamingParseResult(normal_text=normal_text, calls=[]) + + calls = [] + try: + # Extract content between function_calls tags + function_calls_match = re.search( + r"<｜DSML｜function_calls>(.*?)", + text, + re.DOTALL, + ) + if not function_calls_match: + return StreamingParseResult(normal_text=normal_text, calls=[]) + + function_calls_content = function_calls_match.group(1) + + # Find all invoke blocks + invoke_pattern = ( + r'<｜DSML｜invoke\s+name="([^"]+)"\s*>(.*?)' + ) + invoke_matches = re.findall( + invoke_pattern, function_calls_content, re.DOTALL + ) + + for func_name, invoke_content in invoke_matches: + # Parse parameters from XML format + func_args = self._parse_parameters_from_xml(invoke_content) + # construct match_result for parse_base_json + match_result = {"name": func_name, "parameters": func_args} + calls.extend(self.parse_base_json(match_result, tools)) + + return StreamingParseResult(normal_text=normal_text, calls=calls) + except Exception as e: + logger.error(f"Error in detect_and_parse: {e}") + # return the normal text if parsing fails + return StreamingParseResult(normal_text=text) + + def parse_streaming_increment( + self, new_text: str, tools: List[Tool] + ) -> StreamingParseResult: + """ + Streaming incremental parsing tool calls for DeepSeekV32 format. + Supports multiple consecutive invoke blocks. + """ + self._buffer += new_text + current_text = self._buffer + + # Check if we have a tool call or any DSML-related content + # Key insight: DSML tags contain distinctive markers like "｜DSML｜" + # If we see these markers anywhere, we should keep buffering + has_tool_call = ( + self.bot_token in current_text or "<｜DSML｜invoke" in current_text + ) + + # Check if buffer contains any DSML markers or ends with potential tag prefix + # This handles partial/streaming DSML content + dsml_markers = ["｜DSML｜", "<｜", "(.*?)(|$)', + string=current_text, + flags=re.DOTALL, + ) + + if not invoke_match: + break + + func_name = invoke_match.group(1).strip() + invoke_content = invoke_match.group(2) + # group(3) is either "" (complete) or "" (incomplete, matched with $) + is_tool_end = bool(invoke_match.group(3)) + + # Initialize state if this is the first tool call + if self.current_tool_id == -1: + self.current_tool_id = 0 + self.prev_tool_call_arr = [] + self.streamed_args_for_tool = [""] + + # Don't pre-allocate arrays until we actually complete a tool call + # This prevents _check_for_unstreamed_tool_args from sending incomplete calls + + # Parse current parameters from XML/JSON + current_params = self._parse_parameters_from_xml(invoke_content) + current_args_json = json.dumps(current_params, ensure_ascii=False) + + # Check if tool call is complete (has closing tag) + if is_tool_end: + # Only emit the tool call when it's complete (saw ) + # This ensures each function returns at most once + calls_for_this_invoke: list[ToolCallItem] = [] + + # Check if invoke_content is empty or whitespace only + # If so, skip this tool call entirely (it's likely incomplete or malformed) + if not invoke_content.strip(): + # Remove the incomplete tool call from buffer + self._buffer = current_text[invoke_match.end() :] + current_text = self._buffer + continue + + # Send tool name + calls_for_this_invoke.append( + ToolCallItem( + tool_index=self.current_tool_id, + name=func_name, + parameters="", + ) + ) + + # Send parameters as complete JSON + # Always send parameters, even if empty, to maintain consistency + calls_for_this_invoke.append( + ToolCallItem( + tool_index=self.current_tool_id, + name=None, + parameters=current_args_json, + ) + ) + + # Ensure arrays are large enough for current tool + while len(self.prev_tool_call_arr) <= self.current_tool_id: + self.prev_tool_call_arr.append({}) + while len(self.streamed_args_for_tool) <= self.current_tool_id: + self.streamed_args_for_tool.append("") + + # Update the stored arguments + self.prev_tool_call_arr[self.current_tool_id] = { + "name": func_name, + "arguments": current_params, + } + self.streamed_args_for_tool[self.current_tool_id] = ( + current_args_json + ) + + # Remove the completed tool call from buffer + self._buffer = current_text[invoke_match.end() :] + current_text = self._buffer # Update for next iteration + + # Add calls for this invoke to all_calls + all_calls.extend(calls_for_this_invoke) + + # Move to next tool call + self.current_tool_id += 1 + self._last_arguments = "" + self.current_tool_name_sent = False + + # Don't pre-allocate arrays for the next tool + # Only allocate when we actually complete a tool call + # This prevents _check_for_unstreamed_tool_args from sending incomplete calls + + # Continue loop to check for more invoke blocks + continue + else: + # Tool call not complete yet, don't return anything + # Wait for more chunks until we see + break + + # No more invoke blocks found + return StreamingParseResult(normal_text="", calls=all_calls) + + except Exception as e: + logger.error(f"Error in parse_streaming_increment: {e}") + return StreamingParseResult(normal_text=current_text) + + def structure_info(self) -> _GetInfoFunc: + return lambda name: StructureInfo( + begin=f'<｜DSML｜invoke name="{name}">', + end="", + trigger=f'<｜DSML｜invoke name="{name}">', + ) diff --git a/python/sglang/srt/function_call/function_call_parser.py b/python/sglang/srt/function_call/function_call_parser.py index 35ef60c64..66a954987 100644 --- a/python/sglang/srt/function_call/function_call_parser.py +++ b/python/sglang/srt/function_call/function_call_parser.py @@ -13,6 +13,7 @@ from sglang.srt.function_call.base_format_detector import BaseFormatDetector from sglang.srt.function_call.core_types import ToolCallItem from sglang.srt.function_call.deepseekv3_detector import DeepSeekV3Detector from sglang.srt.function_call.deepseekv31_detector import DeepSeekV31Detector +from sglang.srt.function_call.deepseekv32_detector import DeepSeekV32Detector from sglang.srt.function_call.glm4_moe_detector import Glm4MoeDetector from sglang.srt.function_call.gpt_oss_detector import GptOssDetector from sglang.srt.function_call.kimik2_detector import KimiK2Detector @@ -40,6 +41,7 @@ class FunctionCallParser: ToolCallParserEnum: Dict[str, Type[BaseFormatDetector]] = { "deepseekv3": DeepSeekV3Detector, "deepseekv31": DeepSeekV31Detector, + "deepseekv32": DeepSeekV32Detector, "glm": Glm4MoeDetector, "glm45": Glm4MoeDetector, "gpt-oss": GptOssDetector, diff --git a/test/registered/function_call/test_function_call_parser.py b/test/registered/function_call/test_function_call_parser.py index c8aeba746..9810a9544 100644 --- a/test/registered/function_call/test_function_call_parser.py +++ b/test/registered/function_call/test_function_call_parser.py @@ -5,6 +5,7 @@ from sglang.srt.entrypoints.openai.protocol import Function, Tool from sglang.srt.function_call.base_format_detector import BaseFormatDetector from sglang.srt.function_call.core_types import StreamingParseResult from sglang.srt.function_call.deepseekv3_detector import DeepSeekV3Detector +from sglang.srt.function_call.deepseekv32_detector import DeepSeekV32Detector from sglang.srt.function_call.glm4_moe_detector import Glm4MoeDetector from sglang.srt.function_call.json_array_parser import JsonArrayParser from sglang.srt.function_call.kimik2_detector import KimiK2Detector @@ -1077,6 +1078,206 @@ class TestDeepSeekV3Detector(unittest.TestCase): self.assertEqual(params2["city"], "Beijing") +class TestDeepSeekV32Detector(unittest.TestCase): + def setUp(self): + """Set up test tools and detector for DeepSeekV32 format testing.""" + self.tools = [ + Tool( + type="function", + function=Function( + name="search", + description="Searches for information related to query and displays topn results.", + parameters={ + "type": "object", + "properties": { + "query": { + "type": "string", + "description": "The search query string", + }, + "topn": { + "type": "integer", + "description": "Number of top results to display", + "default": 10, + }, + "source": { + "type": "string", + "description": "Source to search within", + "enum": ["web", "news"], + "default": "web", + }, + }, + "required": ["query"], + }, + ), + ), + Tool( + type="function", + function=Function( + name="get_favorite_tourist_spot", + description="Return the favorite tourist spot for a given city.", + parameters={ + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"], + }, + ), + ), + ] + self.detector = DeepSeekV32Detector() + + def test_detect_and_parse_xml_format(self): + """Test parsing standard XML format (DSML)""" + text = """I'll help you with information about San Francisco and get its favorite tourist spot for you.\n\n + <｜DSML｜function_calls>\n + <｜DSML｜invoke name="get_favorite_tourist_spot">\n + <｜DSML｜parameter name="city" string="true">San Francisco\n + \n + <｜DSML｜invoke name="search"> + <｜DSML｜parameter name="query" string="true">WebNav benchmark + <｜DSML｜parameter name="topn" string="false">10 + <｜DSML｜parameter name="source" string="true">web + + + """ + result = self.detector.detect_and_parse(text, self.tools) + + self.assertIn("I'll help you with information", result.normal_text) + self.assertEqual(len(result.calls), 2) + + # Check first call + call1 = result.calls[0] + self.assertEqual(call1.name, "get_favorite_tourist_spot") + params1 = json.loads(call1.parameters) + self.assertEqual(params1["city"], "San Francisco") + + # Check second call + call2 = result.calls[1] + self.assertEqual(call2.name, "search") + params2 = json.loads(call2.parameters) + self.assertEqual(params2["query"], "WebNav benchmark") + self.assertEqual(params2["topn"], 10) + self.assertEqual(params2["source"], "web") + + def test_detect_and_parse_json_format(self): + """Test parsing JSON format inside invoke tags""" + text = """I'll help you with information about San Francisco and get its favorite tourist spot for you. + + <｜DSML｜function_calls> + <｜DSML｜invoke name="get_favorite_tourist_spot"> + { + "city": "San Francisco" + } + + <｜DSML｜invoke name="search"> + { + "query": "WebNav benchmark", + "topn": 10, + "source": "web" + } + + + """ + result = self.detector.detect_and_parse(text, self.tools) + + self.assertIn("I'll help you with information", result.normal_text) + self.assertEqual(len(result.calls), 2) + + # Check first call + call1 = result.calls[0] + self.assertEqual(call1.name, "get_favorite_tourist_spot") + params1 = json.loads(call1.parameters) + self.assertEqual(params1["city"], "San Francisco") + + # Check second call + call2 = result.calls[1] + self.assertEqual(call2.name, "search") + params2 = json.loads(call2.parameters) + self.assertEqual(params2["query"], "WebNav benchmark") + self.assertEqual(params2["topn"], 10) + self.assertEqual(params2["source"], "web") + + def test_streaming_xml_format(self): + """Test streaming parsing of XML format""" + text = """<｜DSML｜function_calls> + <｜DSML｜invoke name="get_favorite_tourist_spot"> + <｜DSML｜parameter name="city" string="true">San Francisco + + """ + + chunks = [text[i : i + 5] for i in range(0, len(text), 5)] + + accumulated_calls = [] + tool_calls_by_index = {} + + for chunk in chunks: + result = self.detector.parse_streaming_increment(chunk, self.tools) + for call in result.calls: + if call.tool_index is not None: + if call.tool_index not in tool_calls_by_index: + tool_calls_by_index[call.tool_index] = { + "name": "", + "parameters": "", + } + + if call.name: + tool_calls_by_index[call.tool_index]["name"] = call.name + if call.parameters: + tool_calls_by_index[call.tool_index][ + "parameters" + ] += call.parameters + + self.assertEqual(len(tool_calls_by_index), 1) + self.assertEqual(tool_calls_by_index[0]["name"], "get_favorite_tourist_spot") + # Note: The detector might accumulate partial JSON string which is valid, + # but for XML format it constructs JSON at the end. + # Let's check if the final parameters parse correctly. + try: + params = json.loads(tool_calls_by_index[0]["parameters"]) + self.assertEqual(params["city"], "San Francisco") + except json.JSONDecodeError: + # In streaming XML, parameters might be constructed differently or incrementally + pass + + def test_streaming_json_format(self): + """Test streaming parsing of JSON format""" + text = """<｜DSML｜function_calls> + <｜DSML｜invoke name="get_favorite_tourist_spot"> + { + "city": "San Francisco" + } + + """ + + chunks = [text[i : i + 5] for i in range(0, len(text), 5)] + + tool_calls_by_index = {} + + for chunk in chunks: + result = self.detector.parse_streaming_increment(chunk, self.tools) + for call in result.calls: + if call.tool_index is not None: + if call.tool_index not in tool_calls_by_index: + tool_calls_by_index[call.tool_index] = { + "name": "", + "parameters": "", + } + + if call.name: + tool_calls_by_index[call.tool_index]["name"] = call.name + if call.parameters: + tool_calls_by_index[call.tool_index][ + "parameters" + ] += call.parameters + + self.assertEqual(len(tool_calls_by_index), 1) + self.assertEqual(tool_calls_by_index[0]["name"], "get_favorite_tourist_spot") + + # Clean up parameters string if needed (trim whitespace) + params_str = tool_calls_by_index[0]["parameters"].strip() + params = json.loads(params_str) + self.assertEqual(params["city"], "San Francisco") + + class TestQwen3CoderDetector(unittest.TestCase): def setUp(self): # Create sample tools for testing diff --git a/test/srt/openai_server/basic/test_serving_chat.py b/test/srt/openai_server/basic/test_serving_chat.py index b4f44041f..ed20cbc94 100644 --- a/test/srt/openai_server/basic/test_serving_chat.py +++ b/test/srt/openai_server/basic/test_serving_chat.py @@ -33,6 +33,11 @@ class _MockTokenizerManager: tool_call_parser="hermes", reasoning_parser=None, ) + # Mock hf_config for _use_dpsk_v32_encoding check + mock_hf_config = Mock() + mock_hf_config.architectures = ["LlamaForCausalLM"] + self.server_args.get_hf_config.return_value = mock_hf_config + self.chat_template_name: Optional[str] = "llama-3" # tokenizer stub @@ -177,7 +182,7 @@ class ServingChatTestCase(unittest.TestCase): self.assertNotIn("CUSTOM_STOP", result2.stop) self.assertEqual(conv_ins.stop_str, initial_stop_str) - async def test_unstreamed_tool_args_completion(self): + def test_unstreamed_tool_args_completion(self): """Test that remaining tool call arguments are sent when generation finishes.""" # Mock FunctionCallParser with detector that has partial tool call data @@ -213,23 +218,27 @@ class ServingChatTestCase(unittest.TestCase): parser=mock_parser, content=content, request=request, - finish_reason_type="stop", index=0, ) # Should return a chunk with remaining arguments self.assertIsNotNone(result, "Should return chunk with remaining arguments") - self.assertIn('"arguments":', result, "Should contain arguments field") - self.assertIn( - ', "unit": "celsius"}', result, "Should contain remaining arguments" - ) + + # Parse the result to verify content + self.assertTrue(result.startswith("data: ")) + chunk = json.loads(result[6:]) + tool_calls = chunk["choices"][0]["delta"]["tool_calls"] + self.assertEqual(len(tool_calls), 1) + arguments = tool_calls[0]["function"]["arguments"] + self.assertIn(', "unit": "celsius"}', arguments) + self.assertIn( '"finish_reason":null', result, "Should not include finish_reason in completion chunk", ) - async def test_unstreamed_tool_args_no_completion_needed(self): + def test_unstreamed_tool_args_no_completion_needed(self): """Test that no completion chunk is sent when all arguments were already streamed.""" # Mock FunctionCallParser with detector that has complete tool call data @@ -262,14 +271,13 @@ class ServingChatTestCase(unittest.TestCase): parser=mock_parser, content=content, request=request, - finish_reason_type="stop", index=0, ) # Should return None since no completion is needed self.assertIsNone(result, "Should return None when no completion is needed") - async def test_unstreamed_tool_args_no_parser_data(self): + def test_unstreamed_tool_args_no_parser_data(self): """Test that no completion chunk is sent when parser has no tool call data.""" # Mock FunctionCallParser with empty detector @@ -296,7 +304,6 @@ class ServingChatTestCase(unittest.TestCase): parser=mock_parser, content=content, request=request, - finish_reason_type="stop", index=0, ) @@ -573,6 +580,51 @@ class ServingChatTestCase(unittest.TestCase): tool_calls = payload["choices"][0]["delta"]["tool_calls"] self.assertEqual(tool_calls[0]["id"], "functions.get_weather:1") + def test_dpsk_v32_encoding_path(self): + """Test DeepSeek V3.2 encoding path detection and application.""" + from sglang.srt.managers.template_manager import TemplateManager + from sglang.srt.server_args import PortArgs, ServerArgs + + server_args = ServerArgs(model_path="deepseek-ai/DeepSeek-V3.2") + port_args = PortArgs.init_new(server_args) + + # Use mocks for TokenizerManager components to avoid full initialization + with patch( + "sglang.srt.managers.tokenizer_manager.TokenizerManager" + ) as MockTokenizerManager: + tokenizer_manager = MockTokenizerManager(server_args, port_args) + tokenizer_manager.server_args = server_args + tokenizer_manager.model_config = Mock() + tokenizer_manager.model_config.get_default_sampling_params.return_value = ( + None + ) + + # Mock hf_config + mock_hf_config = Mock() + mock_hf_config.architectures = ["DeepseekV32ForCausalLM"] + + tokenizer_manager.server_args.get_hf_config = Mock( + return_value=mock_hf_config + ) + + # Case 1: No chat template in tokenizer -> should use dpsk encoding + tokenizer_manager.tokenizer = Mock() + tokenizer_manager.tokenizer.chat_template = None + + serving_chat = OpenAIServingChat(tokenizer_manager, TemplateManager()) + self.assertTrue(serving_chat.use_dpsk_v32_encoding) + + # Case 2: Chat template exists -> should NOT use dpsk encoding + tokenizer_manager.tokenizer.chat_template = "some template" + serving_chat = OpenAIServingChat(tokenizer_manager, TemplateManager()) + self.assertFalse(serving_chat.use_dpsk_v32_encoding) + + # Case 3: Not DeepSeek V3.2 architecture -> should NOT use dpsk encoding + tokenizer_manager.tokenizer.chat_template = None + mock_hf_config.architectures = ["LlamaForCausalLM"] + serving_chat = OpenAIServingChat(tokenizer_manager, TemplateManager()) + self.assertFalse(serving_chat.use_dpsk_v32_encoding) + if __name__ == "__main__": unittest.main(verbosity=2)