chenchenghao/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[Reasoning + Structured Output] make reasoning compatible with structured output (#12551)

Browse Source 
Signed-off-by: Xinyuan Tong <xinyuantong.cs@gmail.com>
Co-authored-by: Xinyuan Tong <xinyuantong.cs@gmail.com>
This commit is contained in:
Muqi Li
2025-12-08 01:28:39 -08:00
committed by GitHub
co-authored by Xinyuan Tong
parent f72a77038f
commit 06836ad02a
8 changed files with 72 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/sglang/srt/constrained/base_grammar_backend.py
+11 -4
View File
@@ -46,6 +46,9 @@ class BaseGrammarObject:
self.grammar_stats = None
self.current_token = None
def maybe_init_reasoning(self, reasoning: bool):
pass
def accept_token(self, token: int) -> None:
"""
Accept a token in the grammar.
@@ -151,7 +154,9 @@ class BaseGrammarBackend:
def dispatch_structural_tag(self, key_string: str) -> Optional[BaseGrammarObject]:
return self._not_supported("structural_tag", key_string)
def _init_value_dispatch(self, key: Tuple[str, str]) -> Optional[BaseGrammarObject]:
def _init_value_dispatch(
self, key: Tuple[str, str], reasoning: bool
) -> Optional[BaseGrammarObject]:
s = time.perf_counter()
key_type, key_string = key
if key_type == "json":
@@ -174,12 +179,14 @@ class BaseGrammarBackend:
return grammar
def get_cached_or_future_value(
self, key: Tuple[str, str]
self, key: Tuple[str, str], reasoning: bool
) -> Optional[BaseGrammarObject]:
value = self.cache.get(key)
if value:
return value.copy(), True
value = self.executor.submit(self._init_value_dispatch, key)
copied_value = value.copy()
copied_value.maybe_init_reasoning(reasoning)
return copied_value, True
value = self.executor.submit(self._init_value_dispatch, key, reasoning)
return value, False
def set_cache(self, key: Tuple[str, str], value: BaseGrammarObject):
python/sglang/srt/constrained/reasoner_grammar_backend.py
+11 -4
View File
@@ -25,7 +25,7 @@ from .base_grammar_backend import (
class ReasonerGrammarObject(BaseGrammarObject):
def __init__(self, grammar: BaseGrammarObject, think_end_id):
def __init__(self, grammar: BaseGrammarObject, think_end_id: int):
super().__init__()
self.grammar = grammar
self.think_end_id = think_end_id
@@ -34,6 +34,9 @@ class ReasonerGrammarObject(BaseGrammarObject):
# + means number of tokens after thinking ended
self.tokens_after_think_end = -1
def maybe_init_reasoning(self, reasoning: bool):
self.tokens_after_think_end = -1 if reasoning else 0
def transfer_state(self, token: int) -> int:
if self.tokens_after_think_end == -1 and token == self.think_end_id:
self.tokens_after_think_end = 0
@@ -109,9 +112,13 @@ class ReasonerGrammarBackend(BaseGrammarBackend):
self.grammar_backend = grammar_backend
self.think_end_id = think_end_id
def _init_value_dispatch(self, key: Tuple[str, str]) -> Optional[BaseGrammarObject]:
ret = self.grammar_backend._init_value_dispatch(key)
def _init_value_dispatch(
self, key: Tuple[str, str], reasoning: bool
) -> Optional[BaseGrammarObject]:
ret = self.grammar_backend._init_value_dispatch(key, reasoning)
# avoid wrapping invalid grammar, so that the scheduler can detect it
if ret is None or ret is INVALID_GRAMMAR_OBJ:
return ret
return ReasonerGrammarObject(ret, self.think_end_id)
obj = ReasonerGrammarObject(ret, self.think_end_id)
obj.maybe_init_reasoning(reasoning)
return obj
python/sglang/srt/entrypoints/openai/serving_chat.py
+23 -24
View File
@@ -213,6 +213,7 @@ class OpenAIServingChat(OpenAIServingBase):
return_hidden_states=request.return_hidden_states,
rid=request.rid,
extra_key=self._compute_extra_key(request),
reasoning=self._get_reasoning_from_request(request),
priority=request.priority,
custom_labels=custom_labels,
custom_logit_processor=request.custom_logit_processor,
@@ -443,7 +444,10 @@ class OpenAIServingChat(OpenAIServingBase):
prompt = prompt[: -len(conv.sep2)]
else:
prompt = conv.get_prompt()
if self._get_enable_thinking_from_request(request):
if self._get_reasoning_from_request(
request
) and self.reasoning_parser not in ["qwen3", "qwen3-thinking", "glm4"]:
# qwen3 and glm4 think internally without a leading <think> token
prompt += "<think>" # Note(Xinyuan): hard code thinking token
image_data = conv.image_data if conv.image_data else None
@@ -775,7 +779,7 @@ class OpenAIServingChat(OpenAIServingBase):
if reasoning_parser and request.separate_reasoning:
is_force_reasoning = (
self.template_manager.force_reasoning
or self._get_enable_thinking_from_request(request)
or self._get_reasoning_from_request(request)
)
try:
parser = ReasoningParser(
@@ -1022,7 +1026,7 @@ class OpenAIServingChat(OpenAIServingBase):
if index not in reasoning_parser_dict:
is_force_reasoning = (
self.template_manager.force_reasoning
or self._get_enable_thinking_from_request(request)
or self._get_reasoning_from_request(request)
)
reasoning_parser_dict[index] = ReasoningParser(
self.reasoning_parser,
@@ -1052,27 +1056,22 @@ class OpenAIServingChat(OpenAIServingBase):
idx += len(list(tool_calls)) if tool_calls is not None else 0 # noqa
return idx
def _get_enable_thinking_from_request(self, request: ChatCompletionRequest) -> bool:
"""Extracts the 'enable_thinking' flag from request chat_template_kwargs.
NOTE: This parameter is only useful for models that support enable_thinking
flag, such as Qwen3.
Args:
request_obj: The request object (or an item from a list of requests).
Returns:
The boolean value of 'enable_thinking' if found, otherwise False.
"""
if hasattr(request, "chat_template_kwargs") and request.chat_template_kwargs:
# For Qwen3 models, `enable_thinking` is supported.
if self.reasoning_parser in ["qwen3", "glm45"]:
return request.chat_template_kwargs.get("enable_thinking", False)
# For DeepSeek-V3.1 models, `thinking` is supported.
elif self.reasoning_parser in ["deepseek-v3"]:
return request.chat_template_kwargs.get("thinking", False)
else:
return False
return False
def _get_reasoning_from_request(self, request: ChatCompletionRequest) -> bool:
"""Judge whether the request needs reasoning"""
if not self.reasoning_parser:
return False
if self.reasoning_parser in ["deepseek-v3"]:
return (
request.chat_template_kwargs is not None
and request.chat_template_kwargs.get("thinking") is True
)
if self.reasoning_parser in ["qwen3", "glm45"]:
# qwen3 and glm45 are reasoning by default
return (
not request.chat_template_kwargs
or request.chat_template_kwargs.get("enable_thinking", True) is True
)
return True # default
async def _process_tool_call_stream(
self,
python/sglang/srt/managers/io_struct.py
+6
View File
@@ -197,6 +197,9 @@ class GenerateReqInput(BaseReq):
bootstrap_room: Optional[Union[List[int], int]] = None
bootstrap_pair_key: Optional[Union[List[str], str]] = None
# For reasoning
reasoning: bool = False
# Validation step duration
validation_time: Optional[float] = None
@@ -675,6 +678,9 @@ class TokenizedGenerateReqInput(BaseReq):
bootstrap_room: Optional[int] = None
bootstrap_pair_key: Optional[str] = None
# For reasoning
reasoning: bool = False
# For data parallel rank routing
data_parallel_rank: Optional[int] = None
python/sglang/srt/managers/schedule_batch.py
+4
View File
@@ -472,6 +472,7 @@ class Req:
token_type_ids: List[int] = None,
session_id: Optional[str] = None,
custom_logit_processor: Optional[str] = None,
reasoning: bool = False,
return_hidden_states: bool = False,
eos_token_ids: Optional[Set[int]] = None,
bootstrap_host: Optional[str] = None,
@@ -517,6 +518,9 @@ class Req:
# For multi-http worker
self.http_worker_ipc = http_worker_ipc
# For reasoning
self.reasoning = reasoning
# Sampling info
if isinstance(sampling_params.custom_params, dict):
sampling_params = copy.copy(sampling_params)
python/sglang/srt/managers/scheduler.py
+4 -1
View File
@@ -1304,6 +1304,7 @@ class Scheduler(
lora_id=recv_req.lora_id,
input_embeds=recv_req.input_embeds,
custom_logit_processor=recv_req.custom_logit_processor,
reasoning=recv_req.reasoning,
return_hidden_states=recv_req.return_hidden_states,
eos_token_ids=self.model_config.hf_eos_token_id,
bootstrap_host=recv_req.bootstrap_host,
@@ -1431,7 +1432,9 @@ class Scheduler(
elif req.sampling_params.structural_tag:
key = ("structural_tag", req.sampling_params.structural_tag)
value, cache_hit = self.grammar_backend.get_cached_or_future_value(key)
value, cache_hit = self.grammar_backend.get_cached_or_future_value(
key, req.reasoning
)
req.grammar = value
if not cache_hit:
python/sglang/srt/managers/tokenizer_manager.py
+1
View File
@@ -887,6 +887,7 @@ class TokenizerManager(TokenizerCommunicatorMixin):
input_embeds=input_embeds,
session_params=session_params,
custom_logit_processor=obj.custom_logit_processor,
reasoning=obj.reasoning,
return_hidden_states=obj.return_hidden_states,
data_parallel_rank=obj.data_parallel_rank,
priority=obj.priority,