diff --git a/python/sglang/srt/constrained/base_grammar_backend.py b/python/sglang/srt/constrained/base_grammar_backend.py index 202e2f6a3..0b6c78416 100644 --- a/python/sglang/srt/constrained/base_grammar_backend.py +++ b/python/sglang/srt/constrained/base_grammar_backend.py @@ -226,17 +226,29 @@ def create_grammar_backend( whitespace_pattern=server_args.constrained_json_whitespace_pattern, ) elif name == "xgrammar": - from sglang.srt.constrained.xgrammar_backend import XGrammarGrammarBackend + from sglang.srt.constrained.xgrammar_backend import ( + TokenizerNotSupportedError, + XGrammarGrammarBackend, + ) # Convert Set[int] to List[int] if needed eos_list = list(eos_token_ids) if eos_token_ids else None - grammar_backend = XGrammarGrammarBackend( - tokenizer, - vocab_size=vocab_size, - model_eos_token_ids=eos_list, - any_whitespace=not server_args.constrained_json_disable_any_whitespace, - ) + try: + grammar_backend = XGrammarGrammarBackend( + tokenizer, + vocab_size=vocab_size, + model_eos_token_ids=eos_list, + any_whitespace=not server_args.constrained_json_disable_any_whitespace, + ) + except TokenizerNotSupportedError as e: + logger.warning( + f"Grammar backend disabled because tokenizer is not supported by XGrammar: {e}. " + "Falling back to grammar_backend='none'. " + "Structured outputs (JSON schema, regex, EBNF) will not be available." + ) + server_args.grammar_backend = "none" + return None elif name == "llguidance": from sglang.srt.constrained.llguidance_backend import GuidanceBackend diff --git a/python/sglang/srt/constrained/xgrammar_backend.py b/python/sglang/srt/constrained/xgrammar_backend.py index 5f2806e3b..fe3f4822a 100644 --- a/python/sglang/srt/constrained/xgrammar_backend.py +++ b/python/sglang/srt/constrained/xgrammar_backend.py @@ -166,6 +166,12 @@ class XGrammarGrammar(BaseGrammarObject): return f"XGrammarGrammar({self.key_string=}, {self.accepted_tokens=}, {self.current_token=})" +class TokenizerNotSupportedError(Exception): + """Raised when tokenizer is not supported by XGrammar backend.""" + + pass + + class XGrammarGrammarBackend(BaseGrammarBackend): def __init__( self, @@ -182,14 +188,21 @@ class XGrammarGrammarBackend(BaseGrammarBackend): if tokenizer_info is None: # Not supported tokenizer - return + raise TokenizerNotSupportedError( + f"Tokenizer type {type(tokenizer).__name__} is not supported by XGrammar" + ) else: # Create TokenizerInfo with model's EOS tokens as the authoritative stop tokens # This ensures consistency between what the model considers EOS and what XGrammar uses - tokenizer_info = TokenizerInfo.from_huggingface( - tokenizer, vocab_size=vocab_size, stop_token_ids=model_eos_token_ids - ) - override_stop_tokens = None + try: + tokenizer_info = TokenizerInfo.from_huggingface( + tokenizer, vocab_size=vocab_size, stop_token_ids=model_eos_token_ids + ) + override_stop_tokens = None + except Exception as e: + raise TokenizerNotSupportedError( + f"Failed to create XGrammar TokenizerInfo from tokenizer: {e}" + ) self.grammar_compiler = GrammarCompiler(tokenizer_info=tokenizer_info) self.vocab_size = vocab_size diff --git a/python/sglang/srt/entrypoints/openai/serving_chat.py b/python/sglang/srt/entrypoints/openai/serving_chat.py index d7eea9d7b..fa2d68240 100644 --- a/python/sglang/srt/entrypoints/openai/serving_chat.py +++ b/python/sglang/srt/entrypoints/openai/serving_chat.py @@ -1069,8 +1069,8 @@ class OpenAIServingChat(OpenAIServingBase): request.chat_template_kwargs is not None and request.chat_template_kwargs.get("thinking") is True ) - if self.reasoning_parser in ["qwen3", "glm45", "nano_v3"]: - # qwen3, glm45, and nano_v3 are reasoning by default + if self.reasoning_parser in ["qwen3", "glm45", "nano_v3", "interns1"]: + # qwen3, glm45, nano_v3, and interns1 are reasoning by default return ( not request.chat_template_kwargs or request.chat_template_kwargs.get("enable_thinking", True) is True diff --git a/python/sglang/srt/function_call/function_call_parser.py b/python/sglang/srt/function_call/function_call_parser.py index 66a954987..ef39571c7 100644 --- a/python/sglang/srt/function_call/function_call_parser.py +++ b/python/sglang/srt/function_call/function_call_parser.py @@ -16,6 +16,7 @@ from sglang.srt.function_call.deepseekv31_detector import DeepSeekV31Detector from sglang.srt.function_call.deepseekv32_detector import DeepSeekV32Detector from sglang.srt.function_call.glm4_moe_detector import Glm4MoeDetector from sglang.srt.function_call.gpt_oss_detector import GptOssDetector +from sglang.srt.function_call.internlm_detector import InternlmDetector from sglang.srt.function_call.kimik2_detector import KimiK2Detector from sglang.srt.function_call.llama32_detector import Llama32Detector from sglang.srt.function_call.minimax_m2 import MinimaxM2Detector @@ -54,6 +55,7 @@ class FunctionCallParser: "qwen3_coder": Qwen3CoderDetector, "step3": Step3Detector, "minimax-m2": MinimaxM2Detector, + "interns1": InternlmDetector, } def __init__(self, tools: List[Tool], tool_call_parser: str): diff --git a/python/sglang/srt/function_call/internlm_detector.py b/python/sglang/srt/function_call/internlm_detector.py new file mode 100644 index 000000000..1781dbc5e --- /dev/null +++ b/python/sglang/srt/function_call/internlm_detector.py @@ -0,0 +1,248 @@ +# modified from https://github.com/InternLM/lmdeploy/blob/main/lmdeploy/serve/openai/tool_parser/internlm2_parser.py + +import json +import logging +import re +from typing import List + +from sglang.srt.entrypoints.openai.protocol import Tool +from sglang.srt.environ import envs +from sglang.srt.function_call.base_format_detector import BaseFormatDetector +from sglang.srt.function_call.core_types import ( + StreamingParseResult, + StructureInfo, + ToolCallItem, + _GetInfoFunc, +) + +logger = logging.getLogger(__name__) + + +class InternlmDetector(BaseFormatDetector): + """ + Detector for InternLM2/Intern-S1 model function call format. + + The InternLM format uses special tokens to delimit function calls + with JSON for arguments. + + Format Structure: + ``` + text<|action_start|> <|plugin|> + {json}<|action_end|> + ``` + + Examples: + ``` + What's the weather like?<|action_start|> <|plugin|> + {"name": "get_weather", "parameters": {"location": "Tokyo"}}<|action_end|> + ``` + + Key Components: + - Tool Call Start: `<|action_start|> <|plugin|>` + - Tool Call End: `<|action_end|>` + - Arguments: JSON object with `name` and `parameters`/`arguments` + - Supports multiple sequential tool calls in both streaming and non-streaming modes + + """ + + def __init__(self): + super().__init__() + self.bot_token = "<|action_start|> <|plugin|>" + self.eot_token = "<|action_end|>" + self.position = 0 + + def has_tool_call(self, text: str) -> bool: + """Check if the text contains an InternLM format tool call.""" + has_call = self.bot_token in text + return has_call + + def get_arguments(self, obj): + """Extract arguments from object, supporting both 'parameters' and 'arguments' keys.""" + if "parameters" in obj: + return obj.get("parameters") + elif "arguments" in obj: + return obj.get("arguments") + return None + + def detect_and_parse(self, text: str, tools: List[Tool]) -> StreamingParseResult: + """ + One-time parsing: Detects and parses tool calls in the provided text. + Supports multiple tool calls in the format: + <|action_start|> <|plugin|>\n{JSON}<|action_end|> + + :param text: The complete text to parse. + :param tools: List of available tools. + :return: StreamingParseResult with normal text and parsed tool calls. + """ + + # Find the first occurrence of tool call marker to extract normal text + idx = text.find(self.bot_token) + normal_text = text[:idx].strip() if idx != -1 else text + + if self.bot_token not in text: + logger.warning("[InternLM Tool Call] No tool call markers found in text") + return StreamingParseResult(normal_text=normal_text, calls=[]) + + # Use regex to find all tool call blocks + # Pattern matches: {self.bot_token}{...}{self.eot_token} + tool_call_pattern = ( + rf"{re.escape(self.bot_token)}\s*(.*?){re.escape(self.eot_token)}" + ) + matches = re.findall(tool_call_pattern, text, re.DOTALL) + + if not matches: + logger.warning("[InternLM Tool Call] No complete tool call blocks found") + return StreamingParseResult(normal_text=text, calls=[]) + + logger.info(f"[InternLM Tool Call] Found {len(matches)} tool call(s)") + + calls = [] + tool_indices = self._get_tool_indices(tools) + + try: + for idx, action_json in enumerate(matches): + action_json = action_json.strip() + + try: + # Parse the JSON + action_dict = json.loads(action_json) + name = action_dict.get("name") + parameters = self.get_arguments(action_dict) + + if not parameters: + parameters = {} + + logger.info( + f"[InternLM Tool Call] Parsed tool call #{idx+1}: name={name}, " + f"parameters={json.dumps(parameters, ensure_ascii=False)}" + ) + + # Validate tool name + if not (name and name in tool_indices): + logger.warning( + f"[InternLM Tool Call] Model attempted to call undefined function: {name}, " + f"available_tools={list(tool_indices.keys())}" + ) + if not envs.SGLANG_FORWARD_UNKNOWN_TOOLS.get(): + continue # Skip this tool call + + # Create tool call item and add to list + tool_call = ToolCallItem( + tool_index=tool_indices[name], + name=name, + parameters=json.dumps(parameters, ensure_ascii=False), + ) + calls.append(tool_call) + + except json.JSONDecodeError as e: + logger.error( + f"[InternLM Tool Call] Failed to parse JSON for tool call #{idx+1}: {e}" + ) + continue + + logger.info( + f"[InternLM Tool Call] Successfully parsed {len(calls)} tool call(s), " + f"normal_text_length={len(normal_text)}" + ) + return StreamingParseResult(normal_text=normal_text, calls=calls) + + except Exception as e: + logger.error( + f"[InternLM Tool Call] Error in detect_and_parse: {e}", exc_info=True + ) + return StreamingParseResult(normal_text=text, calls=[]) + + def parse_streaming_increment( + self, new_text: str, tools: List[Tool] + ) -> StreamingParseResult: + """ + Streaming incremental parsing for InternLM format. + + Supports a single tool call in streaming mode. + """ + self._buffer += new_text + current_text = self._buffer + + # Check if we don't have a tool call start marker + start = current_text.find(self.bot_token) + if start == -1: + # No tool call marker found + # If we've already processed tool calls, don't return text again + if self.current_tool_id > 0: + self._buffer = "" + return StreamingParseResult(normal_text="") + + # Check if buffer could be partial start of bot_token + if not self._ends_with_partial_token(current_text, self.bot_token): + # Not a partial match, return as normal text + normal_text = current_text + self._buffer = "" + # Clean up any stray end tokens + if self.eot_token in normal_text: + normal_text = normal_text.replace(self.eot_token, "") + return StreamingParseResult(normal_text=normal_text) + else: + # Might be partial start token, keep buffering + return StreamingParseResult() + + # Check if we have a complete tool call (with end marker) + end = current_text.find(self.eot_token) + if end != -1: + # We have a complete tool call + # Initialize state if this is the first tool call + if self.current_tool_id == -1: + self.current_tool_id = 0 + self.prev_tool_call_arr = [] + self.streamed_args_for_tool = [""] + + # Ensure we have enough entries in our tracking arrays + while len(self.prev_tool_call_arr) <= self.current_tool_id: + self.prev_tool_call_arr.append({}) + while len(self.streamed_args_for_tool) <= self.current_tool_id: + self.streamed_args_for_tool.append("") + + # Use detect_and_parse on the complete tool call + complete_section = current_text[: end + len(self.eot_token)] + result = self.detect_and_parse(complete_section, tools=tools) + + if result.calls: + # Update the tool call index + result.calls[0].tool_index = self.current_tool_id + # Store the parsed tool call for reference + self.prev_tool_call_arr[self.current_tool_id] = { + "name": result.calls[0].name, + "arguments": json.loads(result.calls[0].parameters), + } + self.streamed_args_for_tool[self.current_tool_id] = result.calls[ + 0 + ].parameters + # Increment tool ID for next tool call + self.current_tool_id += 1 + + # Remove the completed tool call from buffer + self._buffer = current_text[end + len(self.eot_token) :] + return result + + # We have bot_token but no eot_token yet - handle partial tool call streaming + # Extract normal text before the tool call + normal_text = current_text[:start] + # Keep the tool call part in buffer + self._buffer = current_text[start:] + return StreamingParseResult(normal_text=normal_text) + + def structure_info(self) -> _GetInfoFunc: + """ + Return structure information for constrained generation. + + For InternLM format, the structure is: + - begin: <|action_start|> <|plugin|>\n + - end: <|action_end|> + - trigger: the begin token + """ + return lambda name: StructureInfo( + begin='<|action_start|> <|plugin|>\n{"name": "' + + name + + '", "parameters": ', + end="}<|action_end|>", + trigger="<|action_start|> <|plugin|>", + ) diff --git a/python/sglang/srt/parser/reasoning_parser.py b/python/sglang/srt/parser/reasoning_parser.py index 6ed8d9582..8949ba5d7 100644 --- a/python/sglang/srt/parser/reasoning_parser.py +++ b/python/sglang/srt/parser/reasoning_parser.py @@ -314,6 +314,7 @@ class ReasoningParser: "minimax-append-think": MiniMaxAppendThinkDetector, "step3": DeepSeekR1Detector, "nano_v3": NanoV3Detector, + "interns1": Qwen3Detector, } def __init__(