Expose empty chat messages behind conversion failures
The replay failure returns HTTP 400 before generation, so the next diagnostic point must be the chat-template conversion boundary rather than HiCache or scheduler paths. This adds a failure-only summary that records message roles, content shapes, empty indices, tool metadata, bootstrap room, and template mode without logging raw user content. Constraint: Production replay traffic can contain large/private messages, so diagnostics must avoid full content dumps. Rejected: Enable full request logging | too noisy and exposes complete prompt payloads. Rejected: Normalize empty content immediately | would hide whether the malformed message is user, assistant, tool, or template-derived. Confidence: high Scope-risk: narrow Directive: Keep this log failure-only; do not move it to the hot path or print raw content. Tested: python -m py_compile python/sglang/srt/entrypoints/openai/serving_chat.py Tested: remote docker py_compile after scp to g0034:/mnt/beegfs/cjy/sglang-dev Not-tested: Full replay reproduction; requires user-driven traffic after service restart.
This commit is contained in:
laoyao0822
@@ -85,6 +85,19 @@ def _extract_max_dynamic_patch(request: ChatCompletionRequest):
|
||||
return img_max_dynamic_patch, vid_max_dynamic_patch
|
||||
|
||||
|
||||
def _get_message_field(message: Any, field_name: str, default: Any = None) -> Any:
|
||||
if isinstance(message, dict):
|
||||
return message.get(field_name, default)
|
||||
return getattr(message, field_name, default)
|
||||
|
||||
|
||||
def _safe_len(value: Any) -> Optional[int]:
|
||||
try:
|
||||
return len(value)
|
||||
except TypeError:
|
||||
return None
|
||||
|
||||
|
||||
class OpenAIServingChat(OpenAIServingBase):
|
||||
"""Handler for /v1/chat/completions requests"""
|
||||
|
||||
@@ -160,6 +173,130 @@ class OpenAIServingChat(OpenAIServingBase):
|
||||
messages[-1] = {"role": "user", "content": last_content}
|
||||
return messages, assistant_prefix
|
||||
|
||||
@staticmethod
|
||||
def _summarize_content_for_error(content: Any) -> Dict[str, Any]:
|
||||
"""Summarize message content shape without logging user content."""
|
||||
if content is None:
|
||||
return {
|
||||
"kind": "none",
|
||||
"len": None,
|
||||
"stripped_len": None,
|
||||
"empty": True,
|
||||
}
|
||||
|
||||
if isinstance(content, str):
|
||||
stripped_len = len(content.strip())
|
||||
return {
|
||||
"kind": "str",
|
||||
"len": len(content),
|
||||
"stripped_len": stripped_len,
|
||||
"empty": stripped_len == 0,
|
||||
}
|
||||
|
||||
if isinstance(content, list):
|
||||
part_summaries = []
|
||||
total_text_len = 0
|
||||
non_text_parts = 0
|
||||
for part in content[:8]:
|
||||
part_type = _get_message_field(part, "type", type(part).__name__)
|
||||
text = _get_message_field(part, "text", None)
|
||||
text_len = len(text) if isinstance(text, str) else None
|
||||
if text_len is not None:
|
||||
total_text_len += text_len
|
||||
else:
|
||||
non_text_parts += 1
|
||||
part_summaries.append(
|
||||
{
|
||||
"type": part_type,
|
||||
"text_len": text_len,
|
||||
"has_image_url": _get_message_field(part, "image_url", None)
|
||||
is not None,
|
||||
"has_video_url": _get_message_field(part, "video_url", None)
|
||||
is not None,
|
||||
"has_audio_url": _get_message_field(part, "audio_url", None)
|
||||
is not None,
|
||||
}
|
||||
)
|
||||
|
||||
return {
|
||||
"kind": "list",
|
||||
"len": len(content),
|
||||
"stripped_len": None,
|
||||
"empty": len(content) == 0
|
||||
or (total_text_len == 0 and non_text_parts == 0),
|
||||
"total_text_len": total_text_len,
|
||||
"sample_parts": part_summaries,
|
||||
"truncated_parts": max(0, len(content) - len(part_summaries)),
|
||||
}
|
||||
|
||||
return {
|
||||
"kind": type(content).__name__,
|
||||
"len": _safe_len(content),
|
||||
"stripped_len": None,
|
||||
"empty": False,
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def _summarize_message_for_error(
|
||||
cls, index: int, message: Any
|
||||
) -> Dict[str, Any]:
|
||||
content = _get_message_field(message, "content", None)
|
||||
tool_calls = _get_message_field(message, "tool_calls", None)
|
||||
reasoning_content = _get_message_field(message, "reasoning_content", None)
|
||||
|
||||
summary = cls._summarize_content_for_error(content)
|
||||
return {
|
||||
"index": index,
|
||||
"role": _get_message_field(message, "role", None),
|
||||
"content": summary,
|
||||
"has_tool_calls": bool(tool_calls),
|
||||
"tool_call_count": _safe_len(tool_calls) if tool_calls else 0,
|
||||
"has_tool_call_id": _get_message_field(message, "tool_call_id", None)
|
||||
is not None,
|
||||
"has_name": _get_message_field(message, "name", None) is not None,
|
||||
"reasoning_len": len(reasoning_content)
|
||||
if isinstance(reasoning_content, str)
|
||||
else None,
|
||||
}
|
||||
|
||||
def _log_chat_conversion_value_error(
|
||||
self, request: ChatCompletionRequest, stage: str, exc: ValueError
|
||||
) -> None:
|
||||
messages = list(request.messages or [])
|
||||
message_summaries = [
|
||||
self._summarize_message_for_error(i, message)
|
||||
for i, message in enumerate(messages)
|
||||
]
|
||||
empty_message_indices = [
|
||||
item["index"] for item in message_summaries if item["content"]["empty"]
|
||||
]
|
||||
last_message = messages[-1] if messages else None
|
||||
last_role = _get_message_field(last_message, "role", None)
|
||||
last_content = _get_message_field(last_message, "content", None)
|
||||
|
||||
logger.warning(
|
||||
"[OpenAI-chat-conversion-failed] stage=%s error=%r "
|
||||
"rid=%s model=%s stream=%s bootstrap_room=%s "
|
||||
"messages=%d empty_indices=%s last_role=%s "
|
||||
"last_assistant_empty=%s use_dpsk_v32_encoding=%s "
|
||||
"template_name=%s template_content_format=%s summary=%s",
|
||||
stage,
|
||||
str(exc),
|
||||
getattr(request, "rid", None),
|
||||
getattr(request, "model", None),
|
||||
getattr(request, "stream", None),
|
||||
getattr(request, "bootstrap_room", None),
|
||||
len(messages),
|
||||
empty_message_indices,
|
||||
last_role,
|
||||
last_role == "assistant"
|
||||
and self._summarize_content_for_error(last_content)["empty"],
|
||||
self.use_dpsk_v32_encoding,
|
||||
self.template_manager.chat_template_name,
|
||||
self.template_manager.jinja_template_content_format,
|
||||
json.dumps(message_summaries, ensure_ascii=False, separators=(",", ":")),
|
||||
)
|
||||
|
||||
def _append_assistant_prefix_to_prompt_ids(
|
||||
self, prompt_ids: List[int], assistant_prefix: str
|
||||
) -> List[int]:
|
||||
@@ -260,7 +397,11 @@ class OpenAIServingChat(OpenAIServingBase):
|
||||
is_multimodal = self.tokenizer_manager.model_config.is_multimodal
|
||||
|
||||
# Process messages and apply chat template
|
||||
processed_messages = self._process_messages(request, is_multimodal)
|
||||
try:
|
||||
processed_messages = self._process_messages(request, is_multimodal)
|
||||
except ValueError as e:
|
||||
self._log_chat_conversion_value_error(request, "process_messages", e)
|
||||
raise
|
||||
|
||||
# Build sampling parameters
|
||||
sampling_params = request.to_sampling_params(
|
||||
|
||||
Reference in New Issue
Block a user