diff --git a/docs/advanced_features/server_arguments.md b/docs/advanced_features/server_arguments.md index 195178689..0f1a327c9 100644 --- a/docs/advanced_features/server_arguments.md +++ b/docs/advanced_features/server_arguments.md @@ -218,7 +218,7 @@ Please consult the documentation below and [server_args.py](https://github.com/s | `--file-storage-path` | The path of the file storage in backend. | `sglang_storage` | Type: str | | `--enable-cache-report` | Return number of cached tokens in usage.prompt_tokens_details for each openai request. | `False` | bool flag (set to enable) | | `--reasoning-parser` | Specify the parser for reasoning models. Supported parsers: [deepseek-r1, deepseek-v3, glm45, gpt-oss, kimi, qwen3, qwen3-thinking, step3]. | `None` | `deepseek-r1`, `deepseek-v3`, `glm45`, `gpt-oss`, `kimi`, `qwen3`, `qwen3-thinking`, `step3` | -| `--tool-call-parser` | Specify the parser for handling tool-call interactions. Supported parsers: [deepseekv3, deepseekv31, glm, glm45, glm47, gpt-oss, kimi_k2, llama3, mistral, pythonic, qwen, qwen25, qwen3_coder, step3]. | `None` | `deepseekv3`, `deepseekv31`, `glm`, `glm45`, `glm47`, `gpt-oss`, `kimi_k2`, `llama3`, `mistral`, `pythonic`, `qwen`, `qwen25`, `qwen3_coder`, `step3` | +| `--tool-call-parser` | Specify the parser for handling tool-call interactions. Supported parsers: [deepseekv3, deepseekv31, glm, glm45, glm47, gpt-oss, kimi_k2, llama3, mistral, pythonic, qwen, qwen25, qwen3_coder, step3]. | `None` | `deepseekv3`, `deepseekv31`, `glm`, `glm45`, `glm47`, `gpt-oss`, `kimi_k2`, `llama3`, `mistral`, `pythonic`, `qwen`, `qwen25`, `qwen3_coder`, `step3`, `gigachat3` | | `--tool-server` | Either 'demo' or a comma-separated list of tool server urls to use for the model. If not specified, no tool server will be used. | `None` | Type: str | | `--sampling-defaults` | Where to get default sampling parameters. 'openai' uses SGLang/OpenAI defaults (temperature=1.0, top_p=1.0, etc.). 'model' uses the model's generation_config.json to get the recommended sampling parameters if available. Default is 'model'. | `model` | `openai`, `model` | diff --git a/python/sglang/srt/function_call/function_call_parser.py b/python/sglang/srt/function_call/function_call_parser.py index e7fdd6398..2f562192e 100644 --- a/python/sglang/srt/function_call/function_call_parser.py +++ b/python/sglang/srt/function_call/function_call_parser.py @@ -14,6 +14,7 @@ from sglang.srt.function_call.core_types import ToolCallItem from sglang.srt.function_call.deepseekv3_detector import DeepSeekV3Detector from sglang.srt.function_call.deepseekv31_detector import DeepSeekV31Detector from sglang.srt.function_call.deepseekv32_detector import DeepSeekV32Detector +from sglang.srt.function_call.gigachat3_detector import GigaChat3Detector from sglang.srt.function_call.glm4_moe_detector import Glm4MoeDetector from sglang.srt.function_call.glm47_moe_detector import Glm47MoeDetector from sglang.srt.function_call.gpt_oss_detector import GptOssDetector @@ -67,6 +68,7 @@ class FunctionCallParser: "trinity": TrinityDetector, "interns1": InternlmDetector, "hermes": HermesDetector, + "gigachat3": GigaChat3Detector, } def __init__(self, tools: List[Tool], tool_call_parser: str): diff --git a/python/sglang/srt/function_call/gigachat3_detector.py b/python/sglang/srt/function_call/gigachat3_detector.py new file mode 100644 index 000000000..ef94bdf9a --- /dev/null +++ b/python/sglang/srt/function_call/gigachat3_detector.py @@ -0,0 +1,209 @@ +import json +import logging +import re +from typing import List + +from sglang.srt.entrypoints.openai.protocol import Tool +from sglang.srt.function_call.base_format_detector import BaseFormatDetector +from sglang.srt.function_call.core_types import ( + StreamingParseResult, + ToolCallItem, + _GetInfoFunc, +) + +logger = logging.getLogger(__name__) + +REGEX_FUNCTION_CALL = re.compile( + r"function call<\|role_sep\|>\n(.*)", + re.DOTALL, +) + +REGEX_CONTENT_PATTERN = re.compile( + r"^(.*?)<\|message_sep\|>", + re.DOTALL, +) + +NAME_REGEX = re.compile( + r'"name"\s*:\s*"([^"]*)"', + re.DOTALL, +) + +ARGS_REGEX = re.compile( + r'"arguments"\s*:\s*(.*)', + re.DOTALL, +) + + +class GigaChat3Detector(BaseFormatDetector): + def __init__(self) -> None: + super().__init__() + self.tool_started: bool = False + self.tool_name_sent: bool = False + self.end_content: bool = False + self._buffer: str = "" + self.prev_tool_call_arr: list[dict] = [] + + def has_tool_call(self, text: str) -> bool: + """Check if text contains a tool call marker""" + return "function call<|role_sep|>\n" in text + + def detect_and_parse( + self, + text: str, + tools: List[Tool], + ) -> StreamingParseResult: + """ + Non-streaming parsing of complete model output. + Extracts tool calls and content from the full text. + """ + logger.debug(f"[GigaChat3] detect_and_parse: {text}") + model_output = text + function_call = None + content = None + if model_output.rstrip().endswith(""): + model_output = model_output[: model_output.rfind("")] + m_func = REGEX_FUNCTION_CALL.search(model_output) + if m_func: + try: + function_call = json.loads(m_func.group(1), strict=False) + if not ( + isinstance(function_call, dict) + and "name" in function_call + and "arguments" in function_call + ): + function_call = None + elif not isinstance(function_call["arguments"], dict): + function_call = None + except json.JSONDecodeError as e: + logger.warning(f"[GigaChat3] JSON decode error: {e}") + return StreamingParseResult( + normal_text=model_output, + calls=[], + ) + m_content = REGEX_CONTENT_PATTERN.search(model_output) + if m_content: + content = m_content.group(1) + else: + if "<|message_sep|>" in model_output: + content = model_output.split("<|message_sep|>")[0] + else: + content = model_output + if not function_call: + return StreamingParseResult(normal_text=content, calls=[]) + name = function_call["name"] + args = function_call["arguments"] + match_result = {"name": name, "arguments": args} + calls = self.parse_base_json(match_result, tools) + return StreamingParseResult(normal_text=content, calls=calls) + + def parse_streaming_increment( + self, + new_text: str, + tools: List[Tool], + ) -> StreamingParseResult: + """ + Streaming parser for incremental text chunks. + Maintains state across calls to build complete tool calls. + """ + if not new_text: + return StreamingParseResult() + logger.debug(f"[GigaChat3] parse_streaming_increment: '{new_text}'") + self._buffer += new_text + current_text = self._buffer + delta_text = new_text + content = None + func_name = None + cur_args = None + m_func = REGEX_FUNCTION_CALL.search(current_text) + if not self.tool_started: + m_content = REGEX_CONTENT_PATTERN.search(delta_text) + if m_content: + content = m_content.group(1) + self.end_content = True + else: + if "<|message_sep|>" in delta_text: + content = delta_text.split("<|message_sep|>")[0] + self.end_content = True + else: + if not self.end_content: + content = delta_text + if m_func: + self.tool_started = True + logger.debug("[GigaChat3] Tool call started") + if content: + return StreamingParseResult(normal_text=content) + if not m_func: + return StreamingParseResult() + json_tail = m_func.group(1).strip() + name_match = NAME_REGEX.search(json_tail) + if name_match: + func_name = name_match.group(1) + args_match = ARGS_REGEX.search(json_tail) + if args_match: + cur_args = args_match.group(1).strip() + if cur_args.endswith(""): + cur_args = cur_args[: -len("")] + if cur_args.endswith("}"): + try: + candidate = cur_args[:-1].strip() + json.loads(candidate, strict=False) + cur_args = candidate + except json.JSONDecodeError: + pass + calls: List[ToolCallItem] = [] + if not self.prev_tool_call_arr: + self.prev_tool_call_arr.append({}) + if not self.tool_name_sent: + if not func_name: + return StreamingParseResult() + self.tool_name_sent = True + self.prev_tool_call_arr[0]["name"] = func_name + logger.debug(f"[GigaChat3] Sending tool name: {func_name}") + calls.append( + ToolCallItem( + tool_index=0, + name=func_name, + parameters="", + ) + ) + return StreamingParseResult(calls=calls) + if cur_args is None: + return StreamingParseResult() + prev_args = self.prev_tool_call_arr[0].get("arguments_str", "") + if not prev_args: + delta_args = cur_args + elif cur_args.startswith(prev_args): + delta_args = cur_args[len(prev_args) :] + else: + logger.warning( + f"[GigaChat3] Arguments overlap mismatch. " + f"prev='{prev_args[:50]}...' cur='{cur_args[:50]}...'" + ) + return StreamingParseResult() + if not delta_args: + return StreamingParseResult() + self.prev_tool_call_arr[0]["arguments_str"] = cur_args + try: + args_dict = json.loads(cur_args, strict=False) + self.prev_tool_call_arr[0]["arguments"] = args_dict + except json.JSONDecodeError: + self.prev_tool_call_arr[0]["arguments"] = {} + logger.debug(f"[GigaChat3] Sending args delta: '{delta_args[:100]}...'") + calls.append( + ToolCallItem( + tool_index=0, + name=None, + parameters=delta_args, + ) + ) + return StreamingParseResult(calls=calls) + + def supports_structural_tag(self) -> bool: + """GigaChat3 does not use structural tags""" + return False + + def structure_info(self) -> _GetInfoFunc: + """Not applicable for GigaChat3""" + raise NotImplementedError( + "GigaChat3Detector does not support structural_tag format." + ) diff --git a/test/registered/function_call/test_function_call_parser.py b/test/registered/function_call/test_function_call_parser.py index 7263a492c..126d68e5d 100644 --- a/test/registered/function_call/test_function_call_parser.py +++ b/test/registered/function_call/test_function_call_parser.py @@ -6,6 +6,7 @@ from sglang.srt.function_call.base_format_detector import BaseFormatDetector from sglang.srt.function_call.core_types import StreamingParseResult from sglang.srt.function_call.deepseekv3_detector import DeepSeekV3Detector from sglang.srt.function_call.deepseekv32_detector import DeepSeekV32Detector +from sglang.srt.function_call.gigachat3_detector import GigaChat3Detector from sglang.srt.function_call.glm4_moe_detector import Glm4MoeDetector from sglang.srt.function_call.glm47_moe_detector import Glm47MoeDetector from sglang.srt.function_call.json_array_parser import JsonArrayParser @@ -3205,5 +3206,510 @@ class TestLfm2Detector(unittest.TestCase): self.assertEqual(info.trigger, "<|tool_call_start|>") +class TestGigaChat3Detector(unittest.TestCase): + def setUp(self): + self.tools = [ + Tool( + type="function", + function=Function( + name="manage_user_memory", + description="Create, update, or delete a user memory entry.", + parameters={ + "type": "object", + "properties": { + "content": { + "anyOf": [{"type": "string"}, {"type": "null"}], + "default": None, + }, + "action": { + "type": "string", + "enum": ["create", "update", "delete"], + "default": "create", + }, + "id": { + "anyOf": [ + {"type": "string", "format": "uuid"}, + {"type": "null"}, + ], + "default": None, + }, + }, + }, + ), + ), + Tool( + type="function", + function=Function( + name="get_weather", + description="Get weather information", + parameters={ + "type": "object", + "properties": { + "city": {"type": "string", "description": "City name"}, + "unit": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + }, + }, + "required": ["city"], + }, + ), + ), + ] + self.detector = GigaChat3Detector() + + def test_has_tool_call(self): + """Test detection of tool call markers.""" + self.assertTrue(self.detector.has_tool_call("function call<|role_sep|>\n{}")) + self.assertFalse(self.detector.has_tool_call("No tool call here")) + + def test_detect_and_parse_no_tool_call(self): + """Test parsing text without tool calls.""" + text = "How can I help you today?" + result = self.detector.detect_and_parse(text, self.tools) + + self.assertEqual(result.normal_text, text) + self.assertEqual(len(result.calls), 0) + + def test_detect_and_parse_simple_tool_call(self): + """Test parsing a simple tool call without content.""" + text = '<|message_sep|>\n\nfunction call<|role_sep|>\n{"name": "manage_user_memory", "arguments": {"action": "create", "id": "preferences"}}' + result = self.detector.detect_and_parse(text, self.tools) + + # No content before tool call + self.assertEqual(result.normal_text, "") + self.assertEqual(len(result.calls), 1) + self.assertEqual(result.calls[0].name, "manage_user_memory") + + params = json.loads(result.calls[0].parameters) + self.assertEqual(params["action"], "create") + self.assertEqual(params["id"], "preferences") + + def test_detect_and_parse_parameterless_tool_call(self): + """Test parsing a tool call with empty arguments.""" + text = '<|message_sep|>\n\nfunction call<|role_sep|>\n{"name": "manage_user_memory", "arguments": {}}' + result = self.detector.detect_and_parse(text, self.tools) + + self.assertEqual(result.normal_text, "") + self.assertEqual(len(result.calls), 1) + self.assertEqual(result.calls[0].name, "manage_user_memory") + + params = json.loads(result.calls[0].parameters) + self.assertEqual(params, {}) + + def test_detect_and_parse_complex_tool_call(self): + """Test parsing a tool call with nested objects.""" + text = """<|message_sep|> + +function call<|role_sep|> +{"name": "manage_user_memory", "arguments": {"action": "create", "id": "preferences", "content": {"short_answers": true, "hate_emojis": true, "english_ui": false, "russian_math_explanations": true}}}""" + + result = self.detector.detect_and_parse(text, self.tools) + + self.assertEqual(result.normal_text, "") + self.assertEqual(len(result.calls), 1) + self.assertEqual(result.calls[0].name, "manage_user_memory") + + params = json.loads(result.calls[0].parameters) + self.assertEqual(params["action"], "create") + self.assertEqual(params["id"], "preferences") + self.assertIsInstance(params["content"], dict) + self.assertEqual(params["content"]["short_answers"], True) + self.assertEqual(params["content"]["hate_emojis"], True) + + def test_detect_and_parse_with_content_before(self): + """Test parsing tool call with text content before it.""" + text = 'I\'ll check that for you.<|message_sep|>\n\nfunction call<|role_sep|>\n{"name": "manage_user_memory", "arguments": {"action": "create", "id": "preferences"}}' + result = self.detector.detect_and_parse(text, self.tools) + + self.assertEqual(result.normal_text, "I'll check that for you.") + self.assertEqual(len(result.calls), 1) + self.assertEqual(result.calls[0].name, "manage_user_memory") + + def test_detect_and_parse_with_eos_token(self): + """Test parsing tool call with EOS token at the end.""" + text = '<|message_sep|>\n\nfunction call<|role_sep|>\n{"name": "manage_user_memory", "arguments": {"action": "create", "id": "preferences"}}' + result = self.detector.detect_and_parse(text, self.tools) + + self.assertEqual(result.normal_text, "") + self.assertEqual(len(result.calls), 1) + self.assertEqual(result.calls[0].name, "manage_user_memory") + + params = json.loads(result.calls[0].parameters) + self.assertEqual(params["action"], "create") + self.assertEqual(params["id"], "preferences") + + def test_detect_and_parse_with_content_and_eos(self): + """Test parsing tool call with content and EOS token.""" + text = 'I\'ll remember that.<|message_sep|>\n\nfunction call<|role_sep|>\n{"name": "manage_user_memory", "arguments": {"action": "create", "id": "test"}}' + result = self.detector.detect_and_parse(text, self.tools) + + self.assertEqual(result.normal_text, "I'll remember that.") + self.assertEqual(len(result.calls), 1) + self.assertEqual(result.calls[0].name, "manage_user_memory") + + def test_detect_and_parse_invalid_json(self): + """Test parsing with invalid JSON in function call.""" + text = '<|message_sep|>\n\nfunction call<|role_sep|>\n{"name": "manage_user_memory", "arguments": {invalid json}}' + result = self.detector.detect_and_parse(text, self.tools) + + # Should return the full text as content when JSON parsing fails + self.assertIn("function call", result.normal_text) + self.assertEqual(len(result.calls), 0) + + def test_detect_and_parse_missing_name(self): + """Test parsing with missing function name.""" + text = '<|message_sep|>\n\nfunction call<|role_sep|>\n{"arguments": {"action": "create"}}' + result = self.detector.detect_and_parse(text, self.tools) + + # Should not extract tool call if name is missing + self.assertEqual(len(result.calls), 0) + + def test_detect_and_parse_missing_arguments(self): + """Test parsing with missing arguments field.""" + text = '<|message_sep|>\n\nfunction call<|role_sep|>\n{"name": "manage_user_memory"}' + result = self.detector.detect_and_parse(text, self.tools) + + # Should not extract tool call if arguments is missing + self.assertEqual(len(result.calls), 0) + + def test_detect_and_parse_arguments_not_dict(self): + """Test parsing with arguments that is not a dict.""" + text = '<|message_sep|>\n\nfunction call<|role_sep|>\n{"name": "manage_user_memory", "arguments": "string_args"}' + result = self.detector.detect_and_parse(text, self.tools) + + # Should not extract tool call if arguments is not a dict + self.assertEqual(len(result.calls), 0) + + def test_streaming_no_tool_call(self): + """Test streaming text without tool calls.""" + chunks = ["How ", "can ", "I ", "help ", "you?"] + + accumulated_text = "" + for chunk in chunks: + result = self.detector.parse_streaming_increment(chunk, self.tools) + accumulated_text += result.normal_text + + self.assertEqual(accumulated_text, "How can I help you?") + self.assertEqual(len(result.calls), 0) + + def test_streaming_simple_tool_call(self): + """Test streaming a simple tool call.""" + chunks = [ + "<|message_sep|>\n\n", + "function call", + "<|role_sep|>\n", + '{"name": "manage_user_memory", ', + '"arguments": {"action": "create"', + ', "id": "preferences"}}', + ] + + tool_calls_by_index = {} + for chunk in chunks: + result = self.detector.parse_streaming_increment(chunk, self.tools) + + for call in result.calls: + if call.tool_index is not None: + if call.tool_index not in tool_calls_by_index: + tool_calls_by_index[call.tool_index] = { + "name": "", + "parameters": "", + } + + if call.name: + tool_calls_by_index[call.tool_index]["name"] = call.name + if call.parameters: + tool_calls_by_index[call.tool_index][ + "parameters" + ] += call.parameters + + self.assertEqual(len(tool_calls_by_index), 1) + self.assertEqual(tool_calls_by_index[0]["name"], "manage_user_memory") + + params = json.loads(tool_calls_by_index[0]["parameters"]) + self.assertEqual(params["action"], "create") + self.assertEqual(params["id"], "preferences") + + def test_streaming_with_content_before(self): + """Test streaming with content before tool call.""" + chunks = [ + "I'll ", + "help ", + "you.", + "<|message_sep|>\n\n", + "function call", + "<|role_sep|>\n", + '{"name": "get_weather", ', + '"arguments": {"city": "Tokyo"}}', + ] + + accumulated_text = "" + tool_calls_by_index = {} + + for chunk in chunks: + result = self.detector.parse_streaming_increment(chunk, self.tools) + accumulated_text += result.normal_text + + for call in result.calls: + if call.tool_index is not None: + if call.tool_index not in tool_calls_by_index: + tool_calls_by_index[call.tool_index] = { + "name": "", + "parameters": "", + } + + if call.name: + tool_calls_by_index[call.tool_index]["name"] = call.name + if call.parameters: + tool_calls_by_index[call.tool_index][ + "parameters" + ] += call.parameters + + self.assertEqual(accumulated_text, "I'll help you.") + self.assertEqual(len(tool_calls_by_index), 1) + self.assertEqual(tool_calls_by_index[0]["name"], "get_weather") + + params = json.loads(tool_calls_by_index[0]["parameters"]) + self.assertEqual(params["city"], "Tokyo") + + def test_streaming_complex_arguments(self): + """Test streaming with complex nested arguments.""" + chunks = [ + "<|message_sep|>\n\n", + "functi", + "on call<|role_sep|>\n", + '{"name": "manage_user_memory", "arguments": ', + '{"action": "create", "id": "prefs", ', + '"content": {"likes": ["short", "clear"], ', + '"dislikes": ["emojis", "verbose"]}', + "}}", + ] + + tool_calls_by_index = {} + + for chunk in chunks: + result = self.detector.parse_streaming_increment(chunk, self.tools) + + for call in result.calls: + if call.tool_index is not None: + if call.tool_index not in tool_calls_by_index: + tool_calls_by_index[call.tool_index] = { + "name": "", + "parameters": "", + } + + if call.name: + tool_calls_by_index[call.tool_index]["name"] = call.name + if call.parameters: + tool_calls_by_index[call.tool_index][ + "parameters" + ] += call.parameters + + self.assertEqual(len(tool_calls_by_index), 1) + self.assertEqual(tool_calls_by_index[0]["name"], "manage_user_memory") + + params = json.loads(tool_calls_by_index[0]["parameters"]) + self.assertEqual(params["action"], "create") + self.assertEqual(params["content"]["likes"], ["short", "clear"]) + self.assertEqual(params["content"]["dislikes"], ["emojis", "verbose"]) + + def test_streaming_with_eos_token(self): + """Test streaming with EOS token at the end.""" + chunks = [ + "<|message_sep|>\n\n", + "function c", + "all<|role_sep|>\n", + '{"name": "get_weather", ', + '"arguments": {"city": "Paris"}}', + "", + ] + + tool_calls_by_index = {} + + for chunk in chunks: + result = self.detector.parse_streaming_increment(chunk, self.tools) + + for call in result.calls: + if call.tool_index is not None: + if call.tool_index not in tool_calls_by_index: + tool_calls_by_index[call.tool_index] = { + "name": "", + "parameters": "", + } + + if call.name: + tool_calls_by_index[call.tool_index]["name"] = call.name + if call.parameters: + tool_calls_by_index[call.tool_index][ + "parameters" + ] += call.parameters + + self.assertEqual(len(tool_calls_by_index), 1) + self.assertEqual(tool_calls_by_index[0]["name"], "get_weather") + + params = json.loads(tool_calls_by_index[0]["parameters"]) + self.assertEqual(params["city"], "Paris") + + def test_streaming_incomplete_json(self): + """Test streaming with incomplete JSON (no closing brace).""" + chunks = [ + "<|message_sep|>\n\n", + "fun", + "ction call<|role_sep|>\n", + '{"name": "get_weather", ', + '"arguments": {"city": "London"', + # Missing closing braces + ] + + tool_calls_by_index = {} + + for chunk in chunks: + result = self.detector.parse_streaming_increment(chunk, self.tools) + + for call in result.calls: + if call.tool_index is not None: + if call.tool_index not in tool_calls_by_index: + tool_calls_by_index[call.tool_index] = { + "name": "", + "parameters": "", + } + + if call.name: + tool_calls_by_index[call.tool_index]["name"] = call.name + if call.parameters: + tool_calls_by_index[call.tool_index][ + "parameters" + ] += call.parameters + + # Should have name but incomplete parameters + self.assertEqual(len(tool_calls_by_index), 1) + self.assertEqual(tool_calls_by_index[0]["name"], "get_weather") + self.assertTrue(tool_calls_by_index[0]["parameters"].startswith('{"city":')) + + def test_streaming_large_steps(self): + """Test streaming with large chunks that complete in fewer steps.""" + chunks = [ + "I'll remember that.", + "<|message_sep|>\n\nfuncti", + "on call<|role_sep|>\n", + '{"name": "manage_user_memory", "arguments": {"action": "create", "id": "preferences", "content": {"short_answers": true, "hate_emojis": true, ', + '"english_ui": false, "russian_math_explanations": true}', + "}}", + ] + + accumulated_text = "" + tool_calls_by_index = {} + + for chunk in chunks: + result = self.detector.parse_streaming_increment(chunk, self.tools) + accumulated_text += result.normal_text + + for call in result.calls: + if call.tool_index is not None: + if call.tool_index not in tool_calls_by_index: + tool_calls_by_index[call.tool_index] = { + "name": "", + "parameters": "", + } + + if call.name: + tool_calls_by_index[call.tool_index]["name"] = call.name + if call.parameters: + tool_calls_by_index[call.tool_index][ + "parameters" + ] += call.parameters + + self.assertEqual(accumulated_text, "I'll remember that.") + self.assertEqual(len(tool_calls_by_index), 1) + self.assertEqual(tool_calls_by_index[0]["name"], "manage_user_memory") + + params = json.loads(tool_calls_by_index[0]["parameters"]) + self.assertEqual(params["action"], "create") + self.assertEqual(params["content"]["short_answers"], True) + self.assertEqual(params["content"]["russian_math_explanations"], True) + + def test_streaming_very_small_chunks(self): + """Test streaming with very small chunks (character by character).""" + text = '{"name": "get_weather", "arguments": {"city": "NYC"}}' + + # Split into very small chunks (every 5 characters) + chunk_size = 5 + chunked_text = [ + text[i : i + chunk_size] for i in range(0, len(text), chunk_size) + ] + chunks = [ + "<|message_sep|>\n\n", + "func", + "tion call", + "<|role_sep|>\n", + *chunked_text, + ] + tool_calls_by_index = {} + + for chunk in chunks: + result = self.detector.parse_streaming_increment(chunk, self.tools) + + for call in result.calls: + if call.tool_index is not None: + if call.tool_index not in tool_calls_by_index: + tool_calls_by_index[call.tool_index] = { + "name": "", + "parameters": "", + } + + if call.name: + tool_calls_by_index[call.tool_index]["name"] = call.name + if call.parameters: + tool_calls_by_index[call.tool_index][ + "parameters" + ] += call.parameters + + self.assertEqual(len(tool_calls_by_index), 1) + self.assertEqual(tool_calls_by_index[0]["name"], "get_weather") + + params = json.loads(tool_calls_by_index[0]["parameters"]) + self.assertEqual(params["city"], "NYC") + + def test_streaming_json_split_at_quotes(self): + """Test streaming when JSON is split at quote boundaries.""" + chunks = [ + "<|message_sep|>\n\nfunction call<|role_sep|>\n", + '{"name', + '": "', + "get_weather", + '", "arguments', + '": {"city', + '": "', + "Rome", + '"}}', + ] + + tool_calls_by_index = {} + + for chunk in chunks: + result = self.detector.parse_streaming_increment(chunk, self.tools) + + for call in result.calls: + if call.tool_index is not None: + if call.tool_index not in tool_calls_by_index: + tool_calls_by_index[call.tool_index] = { + "name": "", + "parameters": "", + } + + if call.name: + tool_calls_by_index[call.tool_index]["name"] = call.name + if call.parameters: + tool_calls_by_index[call.tool_index][ + "parameters" + ] += call.parameters + + self.assertEqual(len(tool_calls_by_index), 1) + self.assertEqual(tool_calls_by_index[0]["name"], "get_weather") + + params = json.loads(tool_calls_by_index[0]["parameters"]) + self.assertEqual(params["city"], "Rome") + + if __name__ == "__main__": unittest.main()