From ebca5879a18ef7852945ec6f16380a7badd9e489 Mon Sep 17 00:00:00 2001 From: ybyang <10629930+whybeyoung@users.noreply.github.com> Date: Mon, 19 Jan 2026 21:45:39 +0800 Subject: [PATCH] Fix v32 continue_final_message not work (#16567) --- .../srt/entrypoints/openai/serving_chat.py | 91 +++++++++++++++---- 1 file changed, 75 insertions(+), 16 deletions(-) diff --git a/python/sglang/srt/entrypoints/openai/serving_chat.py b/python/sglang/srt/entrypoints/openai/serving_chat.py index 73dbc6d94..1c23e8384 100644 --- a/python/sglang/srt/entrypoints/openai/serving_chat.py +++ b/python/sglang/srt/entrypoints/openai/serving_chat.py @@ -112,6 +112,62 @@ class OpenAIServingChat(OpenAIServingBase): self.use_dpsk_v32_encoding = self._use_dpsk_v32_encoding() + def _handle_last_assistant_message( + self, + messages: List[Dict[str, Any]], + request: ChatCompletionRequest, + ) -> tuple[List[Dict[str, Any]], Optional[str]]: + """ + Handle continue_final_message feature: separate final assistant message. + + If continue_final_message is enabled and the last message is from assistant, + extract its content and remove it from the message list. + If continue_final_message is False and the last message is from assistant, + convert it to a user message to ensure the last message is always from user. + + Only processes text-based content (strings), ignoring multimodal content (lists). + + Args: + messages: List of message dictionaries + request: ChatCompletionRequest with continue_final_message flag + + Returns: + Tuple of (processed_messages, assistant_prefix) + - processed_messages: Messages with last assistant message handled appropriately + - assistant_prefix: Content of the last assistant message (string only), or None + """ + assistant_prefix = None + if messages and messages[-1].get("role") == "assistant": + last_content = messages[-1].get("content") + # Only process string content, ignore multimodal content (lists) + if isinstance(last_content, str): + if request.continue_final_message: + # Extract content and remove the assistant message + assistant_prefix = last_content + messages = messages[:-1] + else: + # Convert the last assistant message to user message + messages[-1] = {"role": "user", "content": last_content} + return messages, assistant_prefix + + def _append_assistant_prefix_to_prompt_ids( + self, prompt_ids: List[int], assistant_prefix: str + ) -> List[int]: + """ + Append assistant prefix to prompt_ids. + + Args: + prompt_ids: Current prompt token IDs + assistant_prefix: Assistant message content to append + + Returns: + Updated prompt_ids with assistant prefix appended + """ + encoded = self.tokenizer_manager.tokenizer.encode(assistant_prefix) + if encoded and encoded[0] == self.tokenizer_manager.tokenizer.bos_token_id: + encoded = encoded[1:] + return prompt_ids + encoded + def _use_dpsk_v32_encoding(self) -> bool: has_chat_template = ( self.tokenizer_manager.tokenizer is not None @@ -327,6 +383,11 @@ class OpenAIServingChat(OpenAIServingBase): messages = request.messages messages = [msg.model_dump() for msg in messages] + # Handle continue_final_message: separate final assistant message + messages, assistant_prefix = self._handle_last_assistant_message( + messages, request + ) + if messages[0]["role"] != "system": # insert an empty system prompt to help render tool system prompt messages.insert(0, {"role": "system", "content": ""}) @@ -334,6 +395,12 @@ class OpenAIServingChat(OpenAIServingBase): messages[0]["tools"] = [tool.model_dump() for tool in request.tools] real_input = encode_messages(messages, thinking_mode=thinking_mode) prompt_ids = self.tokenizer_manager.tokenizer.encode(real_input) + + # Append assistant prefix if continue_final_message is enabled + if assistant_prefix: + prompt_ids = self._append_assistant_prefix_to_prompt_ids( + prompt_ids, assistant_prefix + ) else: for message in request.messages: if message.content is None: @@ -370,15 +437,10 @@ class OpenAIServingChat(OpenAIServingBase): openai_compatible_messages.append(processed_msg) - # Handle assistant prefix for continue_final_message - assistant_prefix = None - if ( - openai_compatible_messages - and openai_compatible_messages[-1]["role"] == "assistant" - ): - if request.continue_final_message: - assistant_prefix = openai_compatible_messages[-1]["content"] - openai_compatible_messages = openai_compatible_messages[:-1] + # Handle continue_final_message: separate final assistant message + openai_compatible_messages, assistant_prefix = ( + self._handle_last_assistant_message(openai_compatible_messages, request) + ) try: prompt_ids = self.tokenizer_manager.tokenizer.apply_chat_template( @@ -420,14 +482,11 @@ class OpenAIServingChat(OpenAIServingBase): # should be treated as client errors (400 BadRequest) raise ValueError(str(template_error)) from template_error + # Append assistant prefix if continue_final_message is enabled if assistant_prefix: - encoded = self.tokenizer_manager.tokenizer.encode(assistant_prefix) - if ( - encoded - and encoded[0] == self.tokenizer_manager.tokenizer.bos_token_id - ): - encoded = encoded[1:] - prompt_ids += encoded + prompt_ids = self._append_assistant_prefix_to_prompt_ids( + prompt_ids, assistant_prefix + ) if is_multimodal: prompt = self.tokenizer_manager.tokenizer.decode(prompt_ids)