From 2212963a6c6086dfde233f94a0b4acd453d5f1a2 Mon Sep 17 00:00:00 2001 From: leavelet Date: Wed, 17 Jun 2026 19:58:42 +0000 Subject: [PATCH] Harden OpenAI tool-call/chat-template + reasoning parsing serving_chat: - Tolerate a malformed historical tool_call `arguments` string (a valid JSON document followed by trailing content) instead of 400-ing the whole multi-turn request: salvage the leading JSON document via raw_decode, else keep the raw string. (A 112-message tool-history request was rejected with orjson "unexpected content after document".) - Catch TypeError (not only jinja2.TemplateError) from the chat-template render so a `tojson` filter on a Jinja Undefined becomes a clean 400 instead of a 500 (upstream #20700 / 5e9bd21979). reasoning_parser: - Strip only LEADING think-start marker tokens; a global replace would delete a `` token that legitimately appears inside reasoning content. Preserve model-generated whitespace in reasoning/normal text (drop .strip()/.rstrip()) (upstream #24251 / dac78768f0). - Add Glm45 detector tests: leading-only strip, token-inside-content preserved, repeated leading markers, streaming trailing whitespace. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../srt/entrypoints/openai/serving_chat.py | 32 +++++++++++++---- python/sglang/srt/parser/reasoning_parser.py | 14 +++++--- .../unit/parser/test_reasoning_parser.py | 34 +++++++++++++++++++ 3 files changed, 70 insertions(+), 10 deletions(-) diff --git a/python/sglang/srt/entrypoints/openai/serving_chat.py b/python/sglang/srt/entrypoints/openai/serving_chat.py index 78a602136..9ea736a28 100644 --- a/python/sglang/srt/entrypoints/openai/serving_chat.py +++ b/python/sglang/srt/entrypoints/openai/serving_chat.py @@ -722,9 +722,28 @@ class OpenAIServingChat(OpenAIServingBase): if "arguments" in item["function"] and isinstance( item["function"]["arguments"], str ): - item["function"]["arguments"] = orjson.loads( - item["function"]["arguments"] - ) + _raw_args = item["function"]["arguments"] + try: + item["function"]["arguments"] = orjson.loads(_raw_args) + except orjson.JSONDecodeError: + # Tolerate a malformed historical tool-call arguments + # string (e.g. a valid JSON document followed by + # trailing content) instead of 400-ing the whole + # multi-turn request: salvage the leading JSON + # document if present, else keep the raw string. + try: + item["function"]["arguments"] = ( + json.JSONDecoder().raw_decode(_raw_args.lstrip())[ + 0 + ] + ) + except ValueError: + logger.warning( + "Keeping unparseable tool_call arguments as a " + "raw string (len=%d) in an assistant history " + "message", + len(_raw_args), + ) openai_compatible_messages.append(processed_msg) @@ -765,9 +784,10 @@ class OpenAIServingChat(OpenAIServingBase): return_dict=False, **extra_template_kwargs, ) - except jinja2.TemplateError as template_error: - # Template errors (e.g., from raise_exception in Jinja templates) - # should be treated as client errors (400 BadRequest) + except (jinja2.TemplateError, TypeError) as template_error: + # Template errors (e.g. raise_exception in Jinja templates) and + # TypeError (e.g. the tojson filter on a Jinja2 Undefined variable) + # should be treated as client errors (400 BadRequest). raise ValueError(str(template_error)) from template_error # Append assistant prefix if continue_final_message is enabled diff --git a/python/sglang/srt/parser/reasoning_parser.py b/python/sglang/srt/parser/reasoning_parser.py index a6867c9f8..ba7f72f9b 100644 --- a/python/sglang/srt/parser/reasoning_parser.py +++ b/python/sglang/srt/parser/reasoning_parser.py @@ -62,7 +62,13 @@ class BaseReasoningFormatDetector: return StreamingParseResult(normal_text=text) # The text is considered to be in a reasoning block. - processed_text = text.replace(self.think_start_token, "").strip() + # Strip only LEADING think-start marker tokens and preserve the rest of the + # generated text verbatim: a global replace would delete a think-start token + # that legitimately appears inside the reasoning content, and .strip() would + # rewrite model-generated whitespace (newlines/indentation). + processed_text = text + while processed_text.startswith(self.think_start_token): + processed_text = processed_text[len(self.think_start_token) :] if ( self.think_end_token not in processed_text @@ -76,7 +82,7 @@ class BaseReasoningFormatDetector: ): # Find the first occurrence of tool_start_token and split there tool_idx = processed_text.find(self.tool_start_token) - reasoning_text = processed_text[:tool_idx].strip() + reasoning_text = processed_text[:tool_idx] # Preserve tool_start_token in normal text normal_text = processed_text[tool_idx:] return StreamingParseResult( @@ -89,7 +95,7 @@ class BaseReasoningFormatDetector: if self.think_end_token in processed_text: splits = processed_text.split(self.think_end_token, maxsplit=1) reasoning_text = splits[0] - normal_text = splits[1].strip() + normal_text = splits[1] return StreamingParseResult( normal_text=normal_text, reasoning_text=reasoning_text @@ -138,7 +144,7 @@ class BaseReasoningFormatDetector: normal_text = current_text[end_idx + len(self.think_end_token) :] return StreamingParseResult( - normal_text=normal_text, reasoning_text=reasoning_text.rstrip() + normal_text=normal_text, reasoning_text=reasoning_text ) # Continue with reasoning content diff --git a/test/registered/unit/parser/test_reasoning_parser.py b/test/registered/unit/parser/test_reasoning_parser.py index 607a6ca52..cb2ec43a5 100644 --- a/test/registered/unit/parser/test_reasoning_parser.py +++ b/test/registered/unit/parser/test_reasoning_parser.py @@ -427,6 +427,40 @@ class TestGlm45Detector(CustomTestCase): self.assertEqual(result.normal_text, text) self.assertEqual(result.reasoning_text, "") + def test_detect_and_parse_preserves_whitespace(self): + """Reasoning + normal text must keep model-generated whitespace (newlines, + indentation); the parser must not rewrite the payload.""" + text = "\n Let me think\n\n\nThe answer is 42.\n" + result = self.detector.detect_and_parse(text) + self.assertEqual(result.reasoning_text, "\n Let me think\n") + self.assertEqual(result.normal_text, "\n\nThe answer is 42.\n") + + def test_detect_and_parse_keeps_think_token_inside_content(self): + """A think-start token appearing INSIDE the reasoning content is real + content, not a marker: only LEADING markers are stripped (a global + replace would corrupt the reasoning).""" + text = "quote the tag literallyanswer" + result = self.detector.detect_and_parse(text) + self.assertEqual(result.reasoning_text, "quote the tag literally") + self.assertEqual(result.normal_text, "answer") + + def test_detect_and_parse_strips_repeated_leading_think_tokens(self): + """Repeated leading think-start tokens are markers, not payload.""" + text = "Let me thinkThe answer is 42." + result = self.detector.detect_and_parse(text) + self.assertEqual(result.reasoning_text, "Let me think") + self.assertEqual(result.normal_text, "The answer is 42.") + + def test_streaming_preserves_reasoning_trailing_whitespace(self): + """Streaming reasoning text must preserve trailing whitespace before the + end token (no rstrip).""" + self.detector.parse_streaming_increment("") + result = self.detector.parse_streaming_increment( + "reasoning \nanswer" + ) + self.assertEqual(result.reasoning_text, "reasoning \n") + self.assertEqual(result.normal_text, "answer") + def test_streaming_normal_flow(self): """Test streaming with normal reasoning flow.""" # Start reasoning