From 71872bb85105e8db3f0752e6e0617412bf8b6f8d Mon Sep 17 00:00:00 2001 From: laoyao0822 Date: Sun, 31 May 2026 04:06:50 +0800 Subject: [PATCH] Expose empty chat messages behind conversion failures The replay failure returns HTTP 400 before generation, so the next diagnostic point must be the chat-template conversion boundary rather than HiCache or scheduler paths. This adds a failure-only summary that records message roles, content shapes, empty indices, tool metadata, bootstrap room, and template mode without logging raw user content. Constraint: Production replay traffic can contain large/private messages, so diagnostics must avoid full content dumps. Rejected: Enable full request logging | too noisy and exposes complete prompt payloads. Rejected: Normalize empty content immediately | would hide whether the malformed message is user, assistant, tool, or template-derived. Confidence: high Scope-risk: narrow Directive: Keep this log failure-only; do not move it to the hot path or print raw content. Tested: python -m py_compile python/sglang/srt/entrypoints/openai/serving_chat.py Tested: remote docker py_compile after scp to g0034:/mnt/beegfs/cjy/sglang-dev Not-tested: Full replay reproduction; requires user-driven traffic after service restart. --- .../srt/entrypoints/openai/serving_chat.py | 143 +++++++++++++++++- 1 file changed, 142 insertions(+), 1 deletion(-) diff --git a/python/sglang/srt/entrypoints/openai/serving_chat.py b/python/sglang/srt/entrypoints/openai/serving_chat.py index 61d467cb8..cc9abc5eb 100644 --- a/python/sglang/srt/entrypoints/openai/serving_chat.py +++ b/python/sglang/srt/entrypoints/openai/serving_chat.py @@ -85,6 +85,19 @@ def _extract_max_dynamic_patch(request: ChatCompletionRequest): return img_max_dynamic_patch, vid_max_dynamic_patch +def _get_message_field(message: Any, field_name: str, default: Any = None) -> Any: + if isinstance(message, dict): + return message.get(field_name, default) + return getattr(message, field_name, default) + + +def _safe_len(value: Any) -> Optional[int]: + try: + return len(value) + except TypeError: + return None + + class OpenAIServingChat(OpenAIServingBase): """Handler for /v1/chat/completions requests""" @@ -160,6 +173,130 @@ class OpenAIServingChat(OpenAIServingBase): messages[-1] = {"role": "user", "content": last_content} return messages, assistant_prefix + @staticmethod + def _summarize_content_for_error(content: Any) -> Dict[str, Any]: + """Summarize message content shape without logging user content.""" + if content is None: + return { + "kind": "none", + "len": None, + "stripped_len": None, + "empty": True, + } + + if isinstance(content, str): + stripped_len = len(content.strip()) + return { + "kind": "str", + "len": len(content), + "stripped_len": stripped_len, + "empty": stripped_len == 0, + } + + if isinstance(content, list): + part_summaries = [] + total_text_len = 0 + non_text_parts = 0 + for part in content[:8]: + part_type = _get_message_field(part, "type", type(part).__name__) + text = _get_message_field(part, "text", None) + text_len = len(text) if isinstance(text, str) else None + if text_len is not None: + total_text_len += text_len + else: + non_text_parts += 1 + part_summaries.append( + { + "type": part_type, + "text_len": text_len, + "has_image_url": _get_message_field(part, "image_url", None) + is not None, + "has_video_url": _get_message_field(part, "video_url", None) + is not None, + "has_audio_url": _get_message_field(part, "audio_url", None) + is not None, + } + ) + + return { + "kind": "list", + "len": len(content), + "stripped_len": None, + "empty": len(content) == 0 + or (total_text_len == 0 and non_text_parts == 0), + "total_text_len": total_text_len, + "sample_parts": part_summaries, + "truncated_parts": max(0, len(content) - len(part_summaries)), + } + + return { + "kind": type(content).__name__, + "len": _safe_len(content), + "stripped_len": None, + "empty": False, + } + + @classmethod + def _summarize_message_for_error( + cls, index: int, message: Any + ) -> Dict[str, Any]: + content = _get_message_field(message, "content", None) + tool_calls = _get_message_field(message, "tool_calls", None) + reasoning_content = _get_message_field(message, "reasoning_content", None) + + summary = cls._summarize_content_for_error(content) + return { + "index": index, + "role": _get_message_field(message, "role", None), + "content": summary, + "has_tool_calls": bool(tool_calls), + "tool_call_count": _safe_len(tool_calls) if tool_calls else 0, + "has_tool_call_id": _get_message_field(message, "tool_call_id", None) + is not None, + "has_name": _get_message_field(message, "name", None) is not None, + "reasoning_len": len(reasoning_content) + if isinstance(reasoning_content, str) + else None, + } + + def _log_chat_conversion_value_error( + self, request: ChatCompletionRequest, stage: str, exc: ValueError + ) -> None: + messages = list(request.messages or []) + message_summaries = [ + self._summarize_message_for_error(i, message) + for i, message in enumerate(messages) + ] + empty_message_indices = [ + item["index"] for item in message_summaries if item["content"]["empty"] + ] + last_message = messages[-1] if messages else None + last_role = _get_message_field(last_message, "role", None) + last_content = _get_message_field(last_message, "content", None) + + logger.warning( + "[OpenAI-chat-conversion-failed] stage=%s error=%r " + "rid=%s model=%s stream=%s bootstrap_room=%s " + "messages=%d empty_indices=%s last_role=%s " + "last_assistant_empty=%s use_dpsk_v32_encoding=%s " + "template_name=%s template_content_format=%s summary=%s", + stage, + str(exc), + getattr(request, "rid", None), + getattr(request, "model", None), + getattr(request, "stream", None), + getattr(request, "bootstrap_room", None), + len(messages), + empty_message_indices, + last_role, + last_role == "assistant" + and self._summarize_content_for_error(last_content)["empty"], + self.use_dpsk_v32_encoding, + self.template_manager.chat_template_name, + self.template_manager.jinja_template_content_format, + json.dumps(message_summaries, ensure_ascii=False, separators=(",", ":")), + ) + def _append_assistant_prefix_to_prompt_ids( self, prompt_ids: List[int], assistant_prefix: str ) -> List[int]: @@ -260,7 +397,11 @@ class OpenAIServingChat(OpenAIServingBase): is_multimodal = self.tokenizer_manager.model_config.is_multimodal # Process messages and apply chat template - processed_messages = self._process_messages(request, is_multimodal) + try: + processed_messages = self._process_messages(request, is_multimodal) + except ValueError as e: + self._log_chat_conversion_value_error(request, "process_messages", e) + raise # Build sampling parameters sampling_params = request.to_sampling_params(