diff --git a/docs/references/para_openai_serving_alignment.md b/docs/references/para_openai_serving_alignment.md new file mode 100644 index 000000000..e09d7bdbf --- /dev/null +++ b/docs/references/para_openai_serving_alignment.md @@ -0,0 +1,159 @@ +# Para OpenAI Serving 对齐说明 + +本文总结当前分支对 `sglang-para` OpenAI API / serving 相关行为的对齐范围、默认策略和启用方式。 + +## 总体策略 + +本分支已经实现 Para 侧高风险 serving 行为，但为了降低耦合，**默认不启用 Para 兼容策略**。需要对齐 Para 业务行为时，通过 server args 显式打开。 + +默认保持上游/本分支原行为： + +- chat 请求不自动填 `max_tokens=32768`。 +- 不强制把 `tool_choice` 改为 `auto`。 +- 不按模型路径自动启用 Kimi / GLM 特化逻辑。 +- 不默认 flatten tool role 的 content list。 +- streaming 不默认预拉首 chunk 做 HTTP error 转换。 + +## 启用 Para 兼容的参数 + +完整启用 Para OpenAI serving 对齐时，可在启动参数中加入： + +```bash +--openai-chat-default-max-tokens 32768 \ +--openai-force-tool-choice-auto \ +--openai-kimi-compat \ +--openai-glm-compat \ +--openai-flatten-tool-role-text-content \ +--openai-streaming-error-preflight +``` + +参数含义： + +| 参数 | 默认值 | 启用后行为 | +| ----------------------------------------- | ------- | ----------------------------------------------------------------------------------------- | +| `--openai-chat-default-max-tokens 32768` | `0` | chat 请求未传 `max_tokens/max_completion_tokens` 时，在 serving 层补默认输出上限。 | +| `--openai-force-tool-choice-auto` | `False` | 将显式非 `auto` 的 chat `tool_choice` 按 `auto` 服务。 | +| `--openai-kimi-compat` | `False` | 根据 `model_path` 识别 Kimi，启用 Kimi `thinking` 映射和固定采样参数。 | +| `--openai-glm-compat` | `False` | 根据 `model_path` 识别 GLM，启用 GLM tool choice 降级、413 超长错误和 GLM usage details。 | +| `--openai-flatten-tool-role-text-content` | `False` | 将 tool role 的纯 text content parts list flatten 成字符串后交给 chat template。 | +| `--openai-streaming-error-preflight` | `False` | streaming 先预拉首 chunk；若首 chunk 是错误，转成普通 HTTP error，而不是 SSE 内报错。 | + +## 已对齐的功能点 + +### 1. Chat 默认输出长度 + +Para 行为：未传 `max_tokens/max_completion_tokens` 时默认 `32768`。 + +当前实现： + +- `ChatCompletionRequest.max_tokens` 仍保持 `None`，避免协议模型硬编码业务默认值。 +- serving 层读取 `openai_chat_default_max_tokens`。 +- 传 `--openai-chat-default-max-tokens 32768` 后，效果与 Para 一致。 + +### 2. Kimi 兼容 + +启用 `--openai-kimi-compat` 后： + +- 支持 chat 请求中的 `thinking` 字段。 +- Kimi 模型下将 `thinking.type != "disabled"` 映射到 `chat_template_kwargs["thinking"]`。 +- 对 Kimi 应用 Para 固定采样参数： + - `top_p=0.95` + - `presence_penalty=0.0` + - `frequency_penalty=0.0` + - `n=1` + +### 3. GLM 兼容 + +启用 `--openai-glm-compat` 后： + +- 根据 `model_path` 包含 `glm` 识别 GLM。 +- GLM 下 `tool_choice="required"` 降级为 `"auto"`。 +- GLM input token 已超过 context length 时，抛 `PayloadTooLargeError`，OpenAI serving 返回 HTTP `413`。 +- GLM chat usage 使用 `completion_tokens_details.reasoning_tokens`。 + +未启用时，GLM 超长输入仍走普通 `ValueError -> HTTP 400` 路径。 + +### 4. Tool choice 全局兼容 + +启用 `--openai-force-tool-choice-auto` 后： + +- chat 请求中显式非 `auto` 的 `tool_choice` 会在 serving 层转为 `auto`。 +- 协议层不再改写 `tool_choice`，便于关闭该行为并降低耦合。 + +### 5. Tool role content flatten + +启用 `--openai-flatten-tool-role-text-content` 后： + +- 对 `role="tool"` 且 content 是纯 text parts list 的消息，将内容拼成字符串。 +- 仅 flatten 纯 text parts；包含其他结构字段的 list 保持原样，避免破坏依赖结构化 tool content 的模板。 + +### 6. GLM function call 参数保留空格 + +无条件对齐 Para： + +- `glm4_moe_detector.py` +- `glm47_moe_detector.py` + +两处 detector 不再对参数值执行 `strip()`，只 strip 参数 key。这样可以保留模型输出或客户端参数值里的前后空格。 + +### 7. `image_url` 字符串兼容 + +无条件对齐 Para： + +- 支持 OpenAI multimodal content part 中 `image_url` 直接传字符串。 +- Pydantic validator 自动转成 `{ "url": ... }`。 + +### 8. Streaming 首包错误转换 + +启用 `--openai-streaming-error-preflight` 后： + +- chat streaming 会先拉取首个 chunk。 +- 如果首 chunk 是 SSE error payload，会转成普通 HTTP error response。 +- 如果首 chunk 正常，会 prepend 回 stream，不影响正常 streaming。 + +### 9. Usage / reasoning tokens + +已对齐 Para usage 聚合能力： + +- `UsageInfo` 增加 `completion_tokens_details`。 +- `UsageProcessor` 聚合 `reasoning_tokens`。 +- Chat/Completion serving 都向 usage processor 传递 reasoning token 计数。 +- 启用 GLM compat 后，GLM chat 使用 `completion_tokens_details.reasoning_tokens`。 + +## 主要改动文件 + +- `python/sglang/srt/server_args.py` +- `python/sglang/srt/entrypoints/openai/protocol.py` +- `python/sglang/srt/entrypoints/openai/serving_chat.py` +- `python/sglang/srt/entrypoints/openai/serving_base.py` +- `python/sglang/srt/entrypoints/openai/usage_processor.py` +- `python/sglang/srt/entrypoints/openai/serving_completions.py` +- `python/sglang/srt/managers/tokenizer_manager.py` +- `python/sglang/srt/function_call/glm4_moe_detector.py` +- `python/sglang/srt/function_call/glm47_moe_detector.py` +- `test/registered/unit/entrypoints/openai/test_para_serving_protocol.py` + +## 验证 + +本地 macOS 使用 uv 虚拟环境完成轻量单测验证： + +```bash +VIRTUAL_ENV=/private/tmp/sglang-para-test-venv \ +PATH=/private/tmp/sglang-para-test-venv/bin:$PATH \ +UV_CACHE_DIR=/private/tmp/uv-cache \ +PYTHONDONTWRITEBYTECODE=1 \ +PYTHONPATH=python:. \ +uv run --active --no-sync python -m unittest discover \ + -s test/registered/unit/entrypoints/openai \ + -p 'test_para_serving_protocol.py' -v +``` + +结果：`Ran 19 tests ... OK`。 + +同时执行： + +```bash +govctl check +``` + +结果：通过。 diff --git a/python/sglang/srt/entrypoints/openai/protocol.py b/python/sglang/srt/entrypoints/openai/protocol.py index 16fa9cbc8..d04ac3581 100644 --- a/python/sglang/srt/entrypoints/openai/protocol.py +++ b/python/sglang/srt/entrypoints/openai/protocol.py @@ -128,6 +128,12 @@ class PromptTokensDetails(BaseModel): cached_tokens: int = 0 +class CompletionTokensDetails(BaseModel): + """Details about completion tokens.""" + + reasoning_tokens: int = 0 + + class UsageInfo(BaseModel): prompt_tokens: int = 0 total_tokens: int = 0 @@ -135,6 +141,7 @@ class UsageInfo(BaseModel): # Used to return cached tokens info when --enable-cache-report is set prompt_tokens_details: Optional[PromptTokensDetails] = None reasoning_tokens: Optional[int] = 0 + completion_tokens_details: Optional[CompletionTokensDetails] = None class StreamOptions(BaseModel): @@ -426,6 +433,13 @@ class ChatCompletionMessageContentImageURL(BaseModel): max_dynamic_patch: Optional[int] = None min_dynamic_patch: Optional[int] = None + @model_validator(mode="before") + @classmethod + def coerce_string(cls, values): + if isinstance(values, str): + return {"url": values} + return values + class ChatCompletionMessageContentVideoURL(BaseModel): url: str @@ -615,6 +629,7 @@ class ChatCompletionRequest(BaseModel): separate_reasoning: bool = True stream_reasoning: bool = True chat_template_kwargs: Optional[Dict] = None + thinking: Optional[Dict] = None # SGLang multimodal tiling controls (extensions) max_dynamic_patch: Optional[int] = None @@ -743,13 +758,16 @@ class ChatCompletionRequest(BaseModel): stop: List[str], model_generation_config: Dict[str, Any], tool_call_constraint: Optional[ToolCallConstraint] = None, + fixed_sampling_overrides: Optional[Dict[str, Any]] = None, ) -> Dict[str, Any]: """ Convert request to sampling parameters. - Priority: user value > model generation_config > OpenAI defaults + Priority: fixed_sampling_overrides (if any) > user value > model_generation_config > OpenAI defaults """ def get_param(param_name: str): + if fixed_sampling_overrides and param_name in fixed_sampling_overrides: + return fixed_sampling_overrides[param_name] value = getattr(self, param_name) if value is None: return model_generation_config.get( @@ -779,7 +797,11 @@ class ChatCompletionRequest(BaseModel): "repetition_penalty": get_param("repetition_penalty"), "regex": self.regex, "ebnf": self.ebnf, - "n": self.n, + "n": ( + fixed_sampling_overrides["n"] + if fixed_sampling_overrides and "n" in fixed_sampling_overrides + else self.n + ), "no_stop_trim": self.no_stop_trim, "ignore_eos": self.ignore_eos, "skip_special_tokens": self.skip_special_tokens, diff --git a/python/sglang/srt/entrypoints/openai/serving_base.py b/python/sglang/srt/entrypoints/openai/serving_base.py index e37ec9dc0..c219a98c5 100644 --- a/python/sglang/srt/entrypoints/openai/serving_base.py +++ b/python/sglang/srt/entrypoints/openai/serving_base.py @@ -14,6 +14,7 @@ from fastapi.responses import ORJSONResponse, StreamingResponse from sglang.srt.entrypoints.openai.encoding_dsv32 import DS32EncodingError from sglang.srt.entrypoints.openai.protocol import ErrorResponse, OpenAIServingRequest from sglang.srt.managers.io_struct import EmbeddingReqInput, GenerateReqInput +from sglang.srt.managers.tokenizer_manager import PayloadTooLargeError from sglang.srt.observability.req_time_stats import monotonic_time from sglang.srt.server_args import ServerArgs @@ -135,6 +136,12 @@ class OpenAIServingBase(ABC): err_type="BadRequest", status_code=400, ) + except PayloadTooLargeError as e: + return self.create_error_response( + message=str(e), + err_type="PayloadTooLargeError", + status_code=413, + ) except DS32EncodingError as e: logger.info(f"DS32EncodingError: {e}") return self.create_error_response( @@ -242,6 +249,52 @@ class OpenAIServingBase(ABC): ) return ORJSONResponse(content=error.model_dump(), status_code=status_code) + def create_error_response_from_first_streaming_chunk( + self, + first_chunk: str, + ) -> Optional[ORJSONResponse]: + if not isinstance(first_chunk, str): + return None + + first_chunk = first_chunk.strip() + if not first_chunk.startswith("data:"): + return None + + data = first_chunk[len("data:") :].strip() + if data == "[DONE]": + return None + + try: + payload = json.loads(data) + except json.JSONDecodeError: + return None + + if not isinstance(payload, dict): + return None + + error = payload.get("error") + if not isinstance(error, dict): + return None + + status_code = ( + error.get("code") + or error.get("status") + or error.get("status_code") + or 500 + ) + if not isinstance(status_code, int) or not 100 <= status_code <= 599: + status_code = 500 + + return self.create_error_response( + message=error.get( + "message", + "Streaming request failed before first chunk.", + ), + err_type=error.get("type", "InternalServerError"), + status_code=status_code, + param=error.get("param"), + ) + def create_streaming_error_response( self, message: str, diff --git a/python/sglang/srt/entrypoints/openai/serving_chat.py b/python/sglang/srt/entrypoints/openai/serving_chat.py index 7f89a4765..008761f1e 100644 --- a/python/sglang/srt/entrypoints/openai/serving_chat.py +++ b/python/sglang/srt/entrypoints/openai/serving_chat.py @@ -48,6 +48,7 @@ from sglang.srt.function_call.function_call_parser import FunctionCallParser from sglang.srt.function_call.json_array_parser import JsonArrayParser from sglang.srt.function_call.utils import get_json_schema_constraint from sglang.srt.managers.io_struct import GenerateReqInput +from sglang.srt.managers.tokenizer_manager import PayloadTooLargeError from sglang.srt.parser.conversation import generate_chat_conv from sglang.srt.parser.jinja_template_utils import process_content_for_template_format from sglang.srt.parser.reasoning_parser import ReasoningParser @@ -133,6 +134,16 @@ class OpenAIServingChat(OpenAIServingBase): and self.tokenizer_manager.model_config.hf_config.model_type == "gpt_oss" ) + # Model-specific Para compatibility is controlled by server args so local + # deployments can opt out without editing OpenAI protocol models. + model_path = self.tokenizer_manager.server_args.model_path.lower() + self.is_kimi = self._get_server_arg("openai_kimi_compat", False) and ( + "kimi" in model_path + ) + self.is_glm = self._get_server_arg("openai_glm_compat", False) and ( + "glm" in model_path + ) + self.use_dpsk_v32_encoding = self._use_dpsk_v32_encoding() def _handle_last_assistant_message( @@ -327,8 +338,42 @@ class OpenAIServingChat(OpenAIServingBase): def _request_id_prefix(self) -> str: return "chatcmpl-" + def _get_server_arg(self, name: str, default: Any) -> Any: + server_args = getattr(self.tokenizer_manager, "server_args", None) + return getattr(server_args, name, default) + + def _apply_openai_serving_defaults( + self, request: ChatCompletionRequest + ) -> None: + if request.max_completion_tokens is not None or request.max_tokens is not None: + return + + default_max_tokens = self._get_server_arg( + "openai_chat_default_max_tokens", 0 + ) + if default_max_tokens and default_max_tokens > 0: + request.max_tokens = default_max_tokens + def _validate_request(self, request: ChatCompletionRequest) -> Optional[str]: """Validate that the input is valid.""" + self._apply_openai_serving_defaults(request) + + if ( + self._get_server_arg("openai_force_tool_choice_auto", False) + and request.tool_choice != "auto" + ): + request.tool_choice = "auto" + + if ( + getattr(self, "is_glm", False) + and isinstance(request.tool_choice, str) + and request.tool_choice.lower() == "required" + ): + logger.warning( + "tool_choice='required' is being downgraded to 'auto' for GLM model." + ) + request.tool_choice = "auto" + if not request.messages: return "Messages cannot be empty." @@ -373,13 +418,50 @@ class OpenAIServingChat(OpenAIServingBase): if schema is None: return "schema_ is required for json_schema response format request." + if getattr(self, "is_kimi", False): + is_think_mode = not ( + request.chat_template_kwargs + and request.chat_template_kwargs.get("thinking") is False + ) + expected_params = self._get_kimi_fixed_params(is_think_mode) + for param, expected_value in expected_params.items(): + user_value = getattr(request, param) + if user_value is not None and abs(user_value - expected_value) >= 1e-3: + return ( + f"Parameter '{param}' cannot be overridden. " + f"Expected: {expected_value}, Got: {user_value}" + ) + + user_temperature = getattr(request, "temperature") + if user_temperature is not None and ( + user_temperature < 0.0 or user_temperature > 1.0 + ): + return ( + "Parameter `temperature` must be in [0, 1.0]. " + f"Got: {user_temperature}" + ) + return None + @staticmethod + def _get_kimi_fixed_params(is_think_mode: bool) -> Dict[str, float]: + """Return Kimi's fixed sampling parameters based on thinking mode.""" + return { + # Keep Para behavior: temperature is validated as a range, but not forced. + # "temperature": 1.0 if is_think_mode else 0.6, + "top_p": 0.95, + "presence_penalty": 0.0, + "frequency_penalty": 0.0, + "n": 1, + } + def _convert_to_internal_request( self, request: ChatCompletionRequest, raw_request: Request = None, ) -> tuple[GenerateReqInput, ChatCompletionRequest]: + self._apply_openai_serving_defaults(request) + reasoning_effort = ( request.chat_template_kwargs.pop("reasoning_effort", None) if request.chat_template_kwargs @@ -404,10 +486,18 @@ class OpenAIServingChat(OpenAIServingBase): raise # Build sampling parameters + fixed_overrides = None + if getattr(self, "is_kimi", False): + is_think_mode = not ( + request.chat_template_kwargs + and request.chat_template_kwargs.get("thinking") is False + ) + fixed_overrides = self._get_kimi_fixed_params(is_think_mode) sampling_params = request.to_sampling_params( stop=processed_messages.stop, model_generation_config=self.default_sampling_params, tool_call_constraint=processed_messages.tool_call_constraint, + fixed_sampling_overrides=fixed_overrides, ) # Handle single vs multiple requests @@ -530,6 +620,13 @@ class OpenAIServingChat(OpenAIServingBase): template_content_format = self.template_manager.jinja_template_content_format + if getattr(self, "is_kimi", False) and request.thinking is not None: + if request.chat_template_kwargs is None: + request.chat_template_kwargs = {} + request.chat_template_kwargs["thinking"] = ( + request.thinking.get("type") != "disabled" + ) + if self.use_dpsk_v32_encoding: thinking_mode = ( "thinking" @@ -587,6 +684,30 @@ class OpenAIServingChat(OpenAIServingBase): modalities, ) + # Normalize tool role content: OpenAI clients may send content as a + # list of content parts, but most chat templates expect a plain + # string for tool messages. Only flatten pure text parts; preserve + # lists that carry tool-semantic fields for templates that iterate + # over those structures. + if ( + self._get_server_arg( + "openai_flatten_tool_role_text_content", False + ) + and processed_msg["role"] == "tool" + and isinstance(processed_msg.get("content"), list) + ): + parts = processed_msg["content"] + is_openai_text_parts = all( + (isinstance(part, dict) and part.get("type") == "text") + or isinstance(part, str) + for part in parts + ) + if is_openai_text_parts: + processed_msg["content"] = "\n".join( + part.get("text", "") if isinstance(part, dict) else part + for part in parts + ) + # per the Transformers docs & maintainers, tool call arguments in # assistant-role messages with tool_calls need to be dicts not JSON str - # this is how tool-use chat templates will expect them moving forwards @@ -745,8 +866,37 @@ class OpenAIServingChat(OpenAIServingBase): raw_request: Request, ) -> StreamingResponse: """Handle streaming chat completion request""" + if not self._get_server_arg("openai_streaming_error_preflight", False): + return StreamingResponse( + self._generate_chat_stream(adapted_request, request, raw_request), + media_type="text/event-stream", + background=self.tokenizer_manager.create_abort_task(adapted_request), + ) + + generator = self._generate_chat_stream(adapted_request, request, raw_request) + + try: + first_chunk = await generator.__anext__() + except PayloadTooLargeError as e: + return self.create_error_response( + str(e), status_code=413, err_type="PayloadTooLargeError" + ) + except ValueError as e: + return self.create_error_response(str(e)) + + first_chunk_error_response = self.create_error_response_from_first_streaming_chunk( + first_chunk + ) + if first_chunk_error_response is not None: + return first_chunk_error_response + + async def prepend_first_chunk(): + yield first_chunk + async for chunk in generator: + yield chunk + return StreamingResponse( - self._generate_chat_stream(adapted_request, request, raw_request), + prepend_first_chunk(), media_type="text/event-stream", background=self.tokenizer_manager.create_abort_task(adapted_request), ) @@ -772,6 +922,7 @@ class OpenAIServingChat(OpenAIServingBase): # Usage tracking prompt_tokens = {} completion_tokens = {} + reasoning_tokens = {} cached_tokens = {} hidden_states = {} routed_experts = {} @@ -786,6 +937,9 @@ class OpenAIServingChat(OpenAIServingBase): completion_tokens[index] = content["meta_info"].get( "completion_tokens", 0 ) + reasoning_tokens[index] = content["meta_info"].get( + "reasoning_tokens", 0 + ) cached_tokens[index] = content["meta_info"].get("cached_tokens", 0) hidden_states[index] = content["meta_info"].get("hidden_states", None) routed_experts[index] = content["meta_info"].get("routed_experts", None) @@ -874,6 +1028,8 @@ class OpenAIServingChat(OpenAIServingBase): chunk.usage = UsageProcessor.calculate_token_usage( prompt_tokens=prompt_tokens.get(index, 0), completion_tokens=completion_tokens.get(index, 0), + reasoning_tokens=reasoning_tokens.get(index, 0), + use_completion_details=self.is_glm, ) yield f"data: {chunk.model_dump_json()}\n\n" @@ -929,6 +1085,8 @@ class OpenAIServingChat(OpenAIServingBase): chunk.usage = UsageProcessor.calculate_token_usage( prompt_tokens=prompt_tokens.get(index, 0), completion_tokens=completion_tokens.get(index, 0), + reasoning_tokens=reasoning_tokens.get(index, 0), + use_completion_details=self.is_glm, ) yield f"data: {chunk.model_dump_json()}\n\n" @@ -1012,6 +1170,8 @@ class OpenAIServingChat(OpenAIServingBase): cached_tokens, n_choices=request.n, enable_cache_report=self.tokenizer_manager.server_args.enable_cache_report, + reasoning_tokens=reasoning_tokens, + use_completion_details=self.is_glm, ) usage_chunk = ChatCompletionStreamResponse( id=content["meta_info"]["id"], @@ -1151,6 +1311,7 @@ class OpenAIServingChat(OpenAIServingBase): ret, n_choices=request.n, enable_cache_report=self.tokenizer_manager.server_args.enable_cache_report, + use_completion_details=self.is_glm, ) return ChatCompletionResponse( @@ -1470,9 +1631,12 @@ class OpenAIServingChat(OpenAIServingBase): if request.stream_options and request.stream_options.continuous_usage_stats: prompt_tokens = content["meta_info"].get("prompt_tokens", 0) completion_tokens = content["meta_info"].get("completion_tokens", 0) + reasoning_tokens = content["meta_info"].get("reasoning_tokens", 0) chunk.usage = UsageProcessor.calculate_token_usage( prompt_tokens=prompt_tokens, completion_tokens=completion_tokens, + reasoning_tokens=reasoning_tokens, + use_completion_details=self.is_glm, ) yield f"data: {chunk.model_dump_json()}\n\n" @@ -1521,9 +1685,12 @@ class OpenAIServingChat(OpenAIServingBase): if request.stream_options and request.stream_options.continuous_usage_stats: prompt_tokens = content["meta_info"].get("prompt_tokens", 0) completion_tokens = content["meta_info"].get("completion_tokens", 0) + reasoning_tokens = content["meta_info"].get("reasoning_tokens", 0) chunk.usage = UsageProcessor.calculate_token_usage( prompt_tokens=prompt_tokens, completion_tokens=completion_tokens, + reasoning_tokens=reasoning_tokens, + use_completion_details=self.is_glm, ) yield f"data: {chunk.model_dump_json()}\n\n" diff --git a/python/sglang/srt/entrypoints/openai/serving_completions.py b/python/sglang/srt/entrypoints/openai/serving_completions.py index a64b488f1..874047c52 100644 --- a/python/sglang/srt/entrypoints/openai/serving_completions.py +++ b/python/sglang/srt/entrypoints/openai/serving_completions.py @@ -204,6 +204,7 @@ class OpenAIServingCompletion(OpenAIServingBase): # Usage tracking prompt_tokens = {} completion_tokens = {} + reasoning_tokens = {} cached_tokens = {} hidden_states = {} routed_experts = {} @@ -219,6 +220,9 @@ class OpenAIServingCompletion(OpenAIServingBase): completion_tokens[index] = content["meta_info"].get( "completion_tokens", 0 ) + reasoning_tokens[index] = content["meta_info"].get( + "reasoning_tokens", 0 + ) cached_tokens[index] = content["meta_info"].get("cached_tokens", 0) hidden_states[index] = content["meta_info"].get("hidden_states", None) routed_experts[index] = content["meta_info"].get("routed_experts", None) @@ -312,6 +316,7 @@ class OpenAIServingCompletion(OpenAIServingBase): chunk.usage = UsageProcessor.calculate_token_usage( prompt_tokens=prompt_tokens.get(index, 0), completion_tokens=completion_tokens.get(index, 0), + reasoning_tokens=reasoning_tokens.get(index, 0), ) yield f"data: {chunk.model_dump_json()}\n\n" @@ -364,6 +369,7 @@ class OpenAIServingCompletion(OpenAIServingBase): cached_tokens, n_choices=request.n, enable_cache_report=self.tokenizer_manager.server_args.enable_cache_report, + reasoning_tokens=reasoning_tokens, ) final_usage_chunk = CompletionStreamResponse( id=content["meta_info"]["id"], diff --git a/python/sglang/srt/entrypoints/openai/usage_processor.py b/python/sglang/srt/entrypoints/openai/usage_processor.py index de88e9eb0..a11c25e92 100644 --- a/python/sglang/srt/entrypoints/openai/usage_processor.py +++ b/python/sglang/srt/entrypoints/openai/usage_processor.py @@ -2,7 +2,11 @@ from __future__ import annotations from typing import Any, Dict, List, Mapping, Optional, final -from sglang.srt.entrypoints.openai.protocol import PromptTokensDetails, UsageInfo +from sglang.srt.entrypoints.openai.protocol import ( + CompletionTokensDetails, + PromptTokensDetails, + UsageInfo, +) @final @@ -19,10 +23,14 @@ class UsageProcessor: responses: List[Dict[str, Any]], n_choices: int = 1, enable_cache_report: bool = False, + use_completion_details: bool = False, ) -> UsageInfo: completion_tokens = sum( r["meta_info"].get("completion_tokens", 0) for r in responses ) + reasoning_tokens = sum( + r["meta_info"].get("reasoning_tokens", 0) for r in responses + ) prompt_tokens = sum( responses[i]["meta_info"].get("prompt_tokens", 0) @@ -41,6 +49,8 @@ class UsageProcessor: prompt_tokens=prompt_tokens, completion_tokens=completion_tokens, cached_tokens=cached_details, + reasoning_tokens=reasoning_tokens, + use_completion_details=use_completion_details, ) @staticmethod @@ -50,12 +60,17 @@ class UsageProcessor: cached_tokens: Mapping[int, int], n_choices: int, enable_cache_report: bool = False, + reasoning_tokens: Optional[Mapping[int, int]] = None, + use_completion_details: bool = False, ) -> UsageInfo: # index % n_choices == 0 marks the first choice of a prompt total_prompt_tokens = sum( tok for idx, tok in prompt_tokens.items() if idx % n_choices == 0 ) total_completion_tokens = sum(completion_tokens.values()) + total_reasoning_tokens = ( + sum(reasoning_tokens.values()) if reasoning_tokens is not None else 0 + ) cached_details = ( UsageProcessor._details_if_cached( @@ -69,6 +84,8 @@ class UsageProcessor: prompt_tokens=total_prompt_tokens, completion_tokens=total_completion_tokens, cached_tokens=cached_details, + reasoning_tokens=total_reasoning_tokens, + use_completion_details=use_completion_details, ) @staticmethod @@ -76,11 +93,27 @@ class UsageProcessor: prompt_tokens: int, completion_tokens: int, cached_tokens: Optional[PromptTokensDetails] = None, + reasoning_tokens: Optional[int] = 0, + use_completion_details: bool = False, ) -> UsageInfo: """Calculate token usage information""" + if use_completion_details: + return UsageInfo( + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + total_tokens=prompt_tokens + completion_tokens, + prompt_tokens_details=cached_tokens, + completion_tokens_details=( + CompletionTokensDetails(reasoning_tokens=reasoning_tokens) + if reasoning_tokens + else None + ), + ) + return UsageInfo( prompt_tokens=prompt_tokens, completion_tokens=completion_tokens, total_tokens=prompt_tokens + completion_tokens, prompt_tokens_details=cached_tokens, + reasoning_tokens=reasoning_tokens, ) diff --git a/python/sglang/srt/function_call/glm47_moe_detector.py b/python/sglang/srt/function_call/glm47_moe_detector.py index 0b25b11eb..9bc11d770 100644 --- a/python/sglang/srt/function_call/glm47_moe_detector.py +++ b/python/sglang/srt/function_call/glm47_moe_detector.py @@ -759,7 +759,6 @@ class Glm47MoeDetector(BaseFormatDetector): arguments = {} for arg_key, arg_value in pairs: arg_key = arg_key.strip() - arg_value = arg_value.strip() arg_type = get_argument_type(func_name, arg_key, tools) parsed_value, is_good_json = parse_arguments(arg_value, arg_type) diff --git a/python/sglang/srt/function_call/glm4_moe_detector.py b/python/sglang/srt/function_call/glm4_moe_detector.py index 0761e24e7..36992b3fe 100644 --- a/python/sglang/srt/function_call/glm4_moe_detector.py +++ b/python/sglang/srt/function_call/glm4_moe_detector.py @@ -613,7 +613,6 @@ class Glm4MoeDetector(BaseFormatDetector): arguments = {} for arg_key, arg_value in pairs: arg_key = arg_key.strip() - arg_value = arg_value.strip() arg_type = get_argument_type(func_name, arg_key, tools) parsed_value, is_good_json = parse_arguments(arg_value, arg_type) diff --git a/python/sglang/srt/managers/tokenizer_manager.py b/python/sglang/srt/managers/tokenizer_manager.py index 3adbfac94..7062d8e68 100644 --- a/python/sglang/srt/managers/tokenizer_manager.py +++ b/python/sglang/srt/managers/tokenizer_manager.py @@ -121,6 +121,11 @@ asyncio.set_event_loop_policy(uvloop.EventLoopPolicy()) _REQUEST_STATE_WAIT_TIMEOUT = envs.SGLANG_REQUEST_STATE_WAIT_TIMEOUT.get() + +class PayloadTooLargeError(Exception): + """Exception raised when a request payload exceeds the model context length.""" + + logger = logging.getLogger(__name__) @@ -806,10 +811,16 @@ class TokenizerManager(TokenizerCommunicatorMixin, TokenizerManagerMultiItemMixi del input_ids[_max_req_len:] input_token_num = len(input_ids) else: - raise ValueError( + error_msg = ( f"The input ({input_token_num} tokens) is longer than the " f"model's context length ({self.context_len} tokens)." ) + if ( + getattr(self.server_args, "openai_glm_compat", False) + and "glm" in self.model_path.lower() + ): + raise PayloadTooLargeError(error_msg) + raise ValueError(error_msg) # Validate total tokens (input + max_new_tokens) max_new_tokens = obj.sampling_params.get("max_new_tokens") @@ -836,6 +847,11 @@ class TokenizerManager(TokenizerCommunicatorMixin, TokenizerManagerMultiItemMixi f"{max_new_tokens} tokens for the completion. Please reduce the number " f"of tokens in the input messages or the completion to fit within the limit." ) + if ( + getattr(self.server_args, "openai_glm_compat", False) + and "glm" in self.model_path.lower() + ): + raise PayloadTooLargeError(error_msg) raise ValueError(error_msg) # Validate embedding requests diff --git a/python/sglang/srt/server_args.py b/python/sglang/srt/server_args.py index cf2a5a2dc..39adfcf3c 100644 --- a/python/sglang/srt/server_args.py +++ b/python/sglang/srt/server_args.py @@ -454,6 +454,12 @@ class ServerArgs: tool_call_parser: Optional[str] = None tool_server: Optional[str] = None sampling_defaults: str = "model" + openai_chat_default_max_tokens: int = 0 + openai_force_tool_choice_auto: bool = False + openai_kimi_compat: bool = False + openai_glm_compat: bool = False + openai_flatten_tool_role_text_content: bool = False + openai_streaming_error_preflight: bool = False # Data parallelism dp_size: int = 1 @@ -4688,6 +4694,66 @@ class ServerArgs: "'model' uses the model's generation_config.json to get the recommended " "sampling parameters if available. Default is 'model'.", ) + parser.add_argument( + "--openai-chat-default-max-tokens", + type=int, + default=ServerArgs.openai_chat_default_max_tokens, + help=( + "Default max_tokens applied to OpenAI chat requests that omit both " + "max_tokens and max_completion_tokens. Set to 0 or a negative value " + "to preserve the upstream unset default." + ), + ) + parser.add_argument( + "--openai-force-tool-choice-auto", + action="store_true", + dest="openai_force_tool_choice_auto", + default=ServerArgs.openai_force_tool_choice_auto, + help=( + "Enable Para-compatible normalization that serves explicit " + "non-auto chat tool_choice values as 'auto'." + ), + ) + parser.add_argument( + "--openai-kimi-compat", + action="store_true", + dest="openai_kimi_compat", + default=ServerArgs.openai_kimi_compat, + help=( + "Enable Para-compatible Kimi handling, including thinking field " + "mapping and fixed Kimi sampling parameter enforcement." + ), + ) + parser.add_argument( + "--openai-glm-compat", + action="store_true", + dest="openai_glm_compat", + default=ServerArgs.openai_glm_compat, + help=( + "Enable Para-compatible GLM handling, including required tool_choice " + "downgrade, 413 context overflow mapping, and GLM usage details." + ), + ) + parser.add_argument( + "--openai-flatten-tool-role-text-content", + action="store_true", + dest="openai_flatten_tool_role_text_content", + default=ServerArgs.openai_flatten_tool_role_text_content, + help=( + "Enable Para-compatible flattening of pure text content-part lists " + "on OpenAI tool-role messages before chat-template rendering." + ), + ) + parser.add_argument( + "--openai-streaming-error-preflight", + action="store_true", + dest="openai_streaming_error_preflight", + default=ServerArgs.openai_streaming_error_preflight, + help=( + "Enable prefetching the first OpenAI streaming chunk to convert " + "pre-stream errors into normal HTTP error responses." + ), + ) # Data parallelism parser.add_argument( diff --git a/test/registered/unit/entrypoints/openai/test_para_serving_protocol.py b/test/registered/unit/entrypoints/openai/test_para_serving_protocol.py new file mode 100644 index 000000000..1e9ecc1f4 --- /dev/null +++ b/test/registered/unit/entrypoints/openai/test_para_serving_protocol.py @@ -0,0 +1,735 @@ +import asyncio +import sys +import types +import unittest +from pathlib import Path +from types import SimpleNamespace + +from pydantic import BaseModel + + +def _install_openai_response_stubs(): + responses_mod = types.ModuleType("openai.types.responses") + response_mod = types.ModuleType("openai.types.responses.response") + tool_mod = types.ModuleType("openai.types.responses.tool") + + class _OpenAIStubModel(BaseModel): + pass + + for name in ( + "ResponseFunctionToolCall", + "ResponseInputItemParam", + "ResponseOutputItem", + "ResponseOutputMessage", + "ResponseOutputText", + "ResponseReasoningItem", + ): + setattr(responses_mod, name, type(name, (_OpenAIStubModel,), {})) + + response_mod.ToolChoice = type("ToolChoice", (_OpenAIStubModel,), {}) + tool_mod.Tool = type("Tool", (_OpenAIStubModel,), {}) + + sys.modules.setdefault("openai", types.ModuleType("openai")) + sys.modules.setdefault("openai.types", types.ModuleType("openai.types")) + sys.modules["openai.types.responses"] = responses_mod + sys.modules["openai.types.responses.response"] = response_mod + sys.modules["openai.types.responses.tool"] = tool_mod + + +_install_openai_response_stubs() + +from sglang.srt.entrypoints.openai.protocol import ChatCompletionRequest +from sglang.srt.entrypoints.openai.usage_processor import UsageProcessor +from sglang.test.ci.ci_register import register_cpu_ci + +register_cpu_ci(est_time=1, suite="stage-a-test-cpu") + +_REPO_ROOT = Path(__file__).resolve().parents[5] + + +class TestParaChatDefaults(unittest.TestCase): + def test_chat_protocol_leaves_omitted_max_tokens_unset(self): + request = ChatCompletionRequest( + model="test-model", + messages=[{"role": "user", "content": "hello"}], + ) + + self.assertIsNone(request.max_tokens) + + def test_serving_applies_configured_para_default_max_tokens(self): + _install_serving_chat_stubs() + from sglang.srt.entrypoints.openai.serving_chat import OpenAIServingChat + + request = ChatCompletionRequest( + model="test-model", + messages=[{"role": "user", "content": "hello"}], + ) + serving = OpenAIServingChat.__new__(OpenAIServingChat) + serving.tokenizer_manager = SimpleNamespace( + server_args=SimpleNamespace(openai_chat_default_max_tokens=32768) + ) + + serving._apply_openai_serving_defaults(request) + + self.assertEqual(request.max_tokens, 32768) + sampling_params = request.to_sampling_params(stop=[], model_generation_config={}) + self.assertEqual(sampling_params["max_new_tokens"], 32768) + + def test_serving_can_disable_para_default_max_tokens(self): + _install_serving_chat_stubs() + from sglang.srt.entrypoints.openai.serving_chat import OpenAIServingChat + + request = ChatCompletionRequest( + model="test-model", + messages=[{"role": "user", "content": "hello"}], + ) + serving = OpenAIServingChat.__new__(OpenAIServingChat) + serving.tokenizer_manager = SimpleNamespace( + server_args=SimpleNamespace(openai_chat_default_max_tokens=0) + ) + + serving._apply_openai_serving_defaults(request) + + self.assertIsNone(request.max_tokens) + + +def _install_serving_chat_stubs(): + """Install lightweight stubs for optional serving dependencies. + + The Para alignment tests run on macOS in a small uv virtualenv. They exercise + request/serving glue without requiring the full SGLang server dependency set. + """ + + import json + + orjson_mod = types.ModuleType("orjson") + orjson_mod.loads = json.loads + orjson_mod.dumps = lambda obj, **_: json.dumps(obj).encode() + sys.modules["orjson"] = orjson_mod + + fastapi_mod = types.ModuleType("fastapi") + fastapi_mod.Request = type("Request", (), {}) + fastapi_mod.HTTPException = type("HTTPException", (Exception,), {}) + responses_mod = types.ModuleType("fastapi.responses") + + class ORJSONResponse: + def __init__(self, content=None, status_code=200, **kwargs): + self.content = content + self.status_code = status_code + self.kwargs = kwargs + + class StreamingResponse: + def __init__(self, content=None, media_type=None, background=None, **kwargs): + self.content = content + self.media_type = media_type + self.background = background + self.kwargs = kwargs + + responses_mod.ORJSONResponse = ORJSONResponse + responses_mod.StreamingResponse = StreamingResponse + sys.modules["fastapi"] = fastapi_mod + sys.modules["fastapi.responses"] = responses_mod + + jsonschema_mod = types.ModuleType("jsonschema") + + class Draft202012Validator: + @staticmethod + def check_schema(_schema): + return None + + jsonschema_mod.Draft202012Validator = Draft202012Validator + jsonschema_mod.SchemaError = type("SchemaError", (Exception,), {}) + sys.modules["jsonschema"] = jsonschema_mod + + serving_base_mod = types.ModuleType("sglang.srt.entrypoints.openai.serving_base") + + class OpenAIServingBase: + def __init__(self, tokenizer_manager): + self.tokenizer_manager = tokenizer_manager + + def create_error_response( + self, message, err_type="BadRequestError", status_code=400, param=None + ): + return ORJSONResponse( + content={ + "message": message, + "type": err_type, + "param": param, + "code": status_code, + }, + status_code=status_code, + ) + + def create_streaming_error_response(self, message): + return json.dumps({"error": {"message": message}}) + + def create_error_response_from_first_streaming_chunk(self, first_chunk): + if not isinstance(first_chunk, str): + return None + first_chunk = first_chunk.strip() + if not first_chunk.startswith("data:"): + return None + data = first_chunk[len("data:") :].strip() + if data == "[DONE]": + return None + try: + payload = json.loads(data) + except json.JSONDecodeError: + return None + error = payload.get("error") if isinstance(payload, dict) else None + if not isinstance(error, dict): + return None + status_code = ( + error.get("code") + or error.get("status") + or error.get("status_code") + or 500 + ) + return self.create_error_response( + message=error.get( + "message", "Streaming request failed before first chunk." + ), + err_type=error.get("type", "InternalServerError"), + status_code=status_code if isinstance(status_code, int) else 500, + param=error.get("param"), + ) + + def extract_custom_labels(self, _raw_request): + return None + + def extract_routed_dp_rank_from_header(self, _raw_request, routed_dp_rank): + return routed_dp_rank + + serving_base_mod.OpenAIServingBase = OpenAIServingBase + sys.modules["sglang.srt.entrypoints.openai.serving_base"] = serving_base_mod + + io_struct_mod = types.ModuleType("sglang.srt.managers.io_struct") + + class GenerateReqInput: + def __init__(self, **kwargs): + self.__dict__.update(kwargs) + + io_struct_mod.GenerateReqInput = GenerateReqInput + io_struct_mod.EmbeddingReqInput = type("EmbeddingReqInput", (), {}) + sys.modules["sglang.srt.managers.io_struct"] = io_struct_mod + + tokenizer_manager_mod = types.ModuleType("sglang.srt.managers.tokenizer_manager") + tokenizer_manager_mod.PayloadTooLargeError = type( + "PayloadTooLargeError", (Exception,), {} + ) + sys.modules["sglang.srt.managers.tokenizer_manager"] = tokenizer_manager_mod + + encoding_mod = types.ModuleType("sglang.srt.entrypoints.openai.encoding_dsv32") + encoding_mod.encode_messages = lambda messages, thinking_mode="chat": str( + (messages, thinking_mode) + ) + encoding_mod.DS32EncodingError = type("DS32EncodingError", (Exception,), {}) + sys.modules["sglang.srt.entrypoints.openai.encoding_dsv32"] = encoding_mod + + utils_mod = types.ModuleType("sglang.srt.entrypoints.openai.utils") + utils_mod.process_cached_tokens_details_from_ret = lambda *_args, **_kwargs: None + utils_mod.process_hidden_states_from_ret = lambda *_args, **_kwargs: None + utils_mod.process_routed_experts_from_ret = lambda *_args, **_kwargs: None + utils_mod.to_openai_style_logprobs = lambda *_args, **_kwargs: None + sys.modules["sglang.srt.entrypoints.openai.utils"] = utils_mod + + core_types_mod = types.ModuleType("sglang.srt.function_call.core_types") + core_types_mod.ToolCallItem = type("ToolCallItem", (), {}) + sys.modules["sglang.srt.function_call.core_types"] = core_types_mod + + function_call_parser_mod = types.ModuleType( + "sglang.srt.function_call.function_call_parser" + ) + function_call_parser_mod.FunctionCallParser = type("FunctionCallParser", (), {}) + sys.modules[ + "sglang.srt.function_call.function_call_parser" + ] = function_call_parser_mod + + json_array_parser_mod = types.ModuleType( + "sglang.srt.function_call.json_array_parser" + ) + json_array_parser_mod.JsonArrayParser = type("JsonArrayParser", (), {}) + sys.modules["sglang.srt.function_call.json_array_parser"] = json_array_parser_mod + + function_utils_mod = types.ModuleType("sglang.srt.function_call.utils") + function_utils_mod.get_json_schema_constraint = lambda *_args, **_kwargs: None + sys.modules["sglang.srt.function_call.utils"] = function_utils_mod + + conversation_mod = types.ModuleType("sglang.srt.parser.conversation") + conversation_mod.generate_chat_conv = lambda *_args, **_kwargs: None + sys.modules["sglang.srt.parser.conversation"] = conversation_mod + + jinja_utils_mod = types.ModuleType("sglang.srt.parser.jinja_template_utils") + jinja_utils_mod.process_content_for_template_format = ( + lambda msg, *_args, **_kwargs: msg + ) + sys.modules["sglang.srt.parser.jinja_template_utils"] = jinja_utils_mod + + reasoning_mod = types.ModuleType("sglang.srt.parser.reasoning_parser") + reasoning_mod.ReasoningParser = type("ReasoningParser", (), {}) + sys.modules["sglang.srt.parser.reasoning_parser"] = reasoning_mod + + +def _install_actual_serving_base_stubs(): + _install_serving_chat_stubs() + + req_time_stats_mod = types.ModuleType( + "sglang.srt.observability.req_time_stats" + ) + req_time_stats_mod.monotonic_time = lambda: 0.0 + sys.modules["sglang.srt.observability.req_time_stats"] = req_time_stats_mod + + server_args_mod = types.ModuleType("sglang.srt.server_args") + server_args_mod.ServerArgs = type("ServerArgs", (), {}) + sys.modules["sglang.srt.server_args"] = server_args_mod + + sys.modules.pop("sglang.srt.entrypoints.openai.serving_base", None) + + +def _install_function_call_stubs(): + import json + from dataclasses import dataclass + + orjson_mod = types.ModuleType("orjson") + orjson_mod.loads = json.loads + orjson_mod.dumps = lambda obj, **_: json.dumps(obj).encode() + sys.modules["orjson"] = orjson_mod + + partial_json_mod = types.ModuleType("partial_json_parser") + partial_json_mod.loads = lambda data, _flags=None: json.loads(data) + sys.modules["partial_json_parser"] = partial_json_mod + + exceptions_mod = types.ModuleType("partial_json_parser.core.exceptions") + exceptions_mod.MalformedJSON = type("MalformedJSON", (Exception,), {}) + sys.modules["partial_json_parser.core.exceptions"] = exceptions_mod + + options_mod = types.ModuleType("partial_json_parser.core.options") + + class Allow: + STR = 1 + OBJ = 2 + ARR = 4 + ALL = STR | OBJ | ARR + + options_mod.Allow = Allow + sys.modules["partial_json_parser.core.options"] = options_mod + + core_types_mod = types.ModuleType("sglang.srt.function_call.core_types") + + class ToolCallItem(BaseModel): + tool_index: int + name: str | None = None + parameters: str + + class StreamingParseResult(BaseModel): + normal_text: str = "" + calls: list[ToolCallItem] = [] + + @dataclass + class StructureInfo: + begin: str + end: str + trigger: str + + core_types_mod.ToolCallItem = ToolCallItem + core_types_mod.StreamingParseResult = StreamingParseResult + core_types_mod.StructureInfo = StructureInfo + core_types_mod._GetInfoFunc = object + sys.modules["sglang.srt.function_call.core_types"] = core_types_mod + + function_utils_mod = types.ModuleType("sglang.srt.function_call.utils") + function_utils_mod._find_common_prefix = lambda left, right: "" + function_utils_mod._is_complete_json = lambda data: True + function_utils_mod._partial_json_loads = ( + lambda data, _flags=None: (json.loads(data), len(data)) + ) + function_utils_mod.infer_type_from_json_schema = ( + lambda schema: schema.get("type") if isinstance(schema, dict) else None + ) + function_utils_mod.get_json_schema_constraint = lambda *_args, **_kwargs: None + sys.modules["sglang.srt.function_call.utils"] = function_utils_mod + + +class TestParaKimiAlignment(unittest.TestCase): + def test_chat_accepts_thinking_and_fixed_sampling_overrides_win(self): + request = ChatCompletionRequest( + model="kimi-test-model", + messages=[{"role": "user", "content": "hello"}], + thinking={"type": "disabled"}, + top_p=0.1, + n=2, + ) + + self.assertEqual(request.thinking, {"type": "disabled"}) + sampling_params = request.to_sampling_params( + stop=[], + model_generation_config={"top_p": 0.7}, + fixed_sampling_overrides={"top_p": 0.95, "n": 1}, + ) + self.assertEqual(sampling_params["top_p"], 0.95) + self.assertEqual(sampling_params["n"], 1) + + def test_kimi_thinking_maps_to_chat_template_kwargs(self): + _install_serving_chat_stubs() + from sglang.srt.entrypoints.openai.serving_chat import OpenAIServingChat + + captured = {} + + class FakeTokenizer: + def apply_chat_template(self, messages, **kwargs): + captured["messages"] = messages + captured["kwargs"] = kwargs + return [1, 2, 3] + + serving = OpenAIServingChat.__new__(OpenAIServingChat) + serving.is_kimi = True + serving.use_dpsk_v32_encoding = False + serving.template_manager = SimpleNamespace( + jinja_template_content_format="openai" + ) + serving.tokenizer_manager = SimpleNamespace(tokenizer=FakeTokenizer()) + + request = ChatCompletionRequest( + model="kimi-test-model", + messages=[{"role": "user", "content": "hello"}], + thinking={"type": "disabled"}, + ) + + serving._apply_jinja_template(request, tools=None, is_multimodal=False) + + self.assertEqual(request.chat_template_kwargs, {"thinking": False}) + self.assertEqual(captured["kwargs"]["thinking"], False) + + +class TestParaGlmAlignment(unittest.TestCase): + def test_serving_chat_detects_glm_model_path(self): + _install_serving_chat_stubs() + from sglang.srt.entrypoints.openai.serving_chat import OpenAIServingChat + + tokenizer_manager = SimpleNamespace( + server_args=SimpleNamespace( + tool_call_parser=None, + reasoning_parser=None, + model_path="/models/GLM-4.5", + openai_glm_compat=True, + openai_kimi_compat=False, + ), + model_config=SimpleNamespace( + get_default_sampling_params=lambda: {}, + hf_config=SimpleNamespace(model_type="glm", architectures=[]), + ), + tokenizer=SimpleNamespace(chat_template="template"), + ) + + serving = OpenAIServingChat( + tokenizer_manager, + template_manager=SimpleNamespace(jinja_template_content_format="openai"), + ) + + self.assertTrue(serving.is_glm) + self.assertFalse(serving.is_kimi) + + def test_glm_required_tool_choice_is_downgraded_to_auto(self): + _install_serving_chat_stubs() + from sglang.srt.entrypoints.openai.serving_chat import OpenAIServingChat + + serving = OpenAIServingChat.__new__(OpenAIServingChat) + serving.is_glm = True + serving.is_kimi = False + serving.tokenizer_manager = SimpleNamespace( + server_args=SimpleNamespace( + context_length=65536, + allow_auto_truncate=False, + openai_force_tool_choice_auto=False, + openai_chat_default_max_tokens=32768, + ) + ) + + request = ChatCompletionRequest( + model="glm-test-model", + messages=[{"role": "user", "content": "hello"}], + tools=[ + { + "type": "function", + "function": {"name": "lookup", "parameters": {"type": "object"}}, + } + ], + tool_choice="required", + ) + error = serving._validate_request(request) + + self.assertIsNone(error) + self.assertEqual(request.tool_choice, "auto") + + +class TestParaToolChoiceAlignment(unittest.TestCase): + def test_protocol_preserves_explicit_non_auto_tool_choice(self): + required_request = ChatCompletionRequest( + model="test-model", + messages=[{"role": "user", "content": "hello"}], + tools=[ + { + "type": "function", + "function": {"name": "lookup", "parameters": {"type": "object"}}, + } + ], + tool_choice="required", + ) + none_request = ChatCompletionRequest( + model="test-model", + messages=[{"role": "user", "content": "hello"}], + tool_choice="none", + ) + + self.assertEqual(required_request.tool_choice, "required") + self.assertEqual(none_request.tool_choice, "none") + + def test_serving_arg_coerces_explicit_non_auto_tool_choice_to_auto(self): + _install_serving_chat_stubs() + from sglang.srt.entrypoints.openai.serving_chat import OpenAIServingChat + + serving = OpenAIServingChat.__new__(OpenAIServingChat) + serving.is_glm = False + serving.is_kimi = False + serving.tokenizer_manager = SimpleNamespace( + server_args=SimpleNamespace( + context_length=65536, + allow_auto_truncate=False, + openai_force_tool_choice_auto=True, + openai_chat_default_max_tokens=32768, + ) + ) + request = ChatCompletionRequest( + model="test-model", + messages=[{"role": "user", "content": "hello"}], + tool_choice="none", + ) + + error = serving._validate_request(request) + + self.assertIsNone(error) + self.assertEqual(request.tool_choice, "auto") + + +class TestParaToolCallAlignment(unittest.TestCase): + def test_tool_role_text_content_parts_are_flattened_for_chat_templates(self): + _install_serving_chat_stubs() + from sglang.srt.entrypoints.openai.serving_chat import OpenAIServingChat + + captured = {} + + class FakeTokenizer: + def apply_chat_template(self, messages, **kwargs): + captured["messages"] = messages + captured["kwargs"] = kwargs + return [1, 2, 3] + + serving = OpenAIServingChat.__new__(OpenAIServingChat) + serving.is_kimi = False + serving.use_dpsk_v32_encoding = False + serving.template_manager = SimpleNamespace( + jinja_template_content_format="openai" + ) + serving.tokenizer_manager = SimpleNamespace( + tokenizer=FakeTokenizer(), + server_args=SimpleNamespace(openai_flatten_tool_role_text_content=True), + ) + + request = ChatCompletionRequest( + model="test-model", + messages=[ + { + "role": "tool", + "tool_call_id": "call-1", + "content": [ + {"type": "text", "text": "first line"}, + {"type": "text", "text": "second line"}, + ], + } + ], + ) + + serving._apply_jinja_template(request, tools=None, is_multimodal=False) + + self.assertEqual( + captured["messages"][0]["content"], + "first line\nsecond line", + ) + + def test_glm_detectors_preserve_argument_value_whitespace(self): + _install_function_call_stubs() + from sglang.srt.function_call.glm4_moe_detector import Glm4MoeDetector + from sglang.srt.function_call.glm47_moe_detector import Glm47MoeDetector + + for detector_cls in (Glm4MoeDetector, Glm47MoeDetector): + with self.subTest(detector=detector_cls.__name__): + arguments = detector_cls()._parse_argument_pairs( + [(" payload ", " keep surrounding spaces ")], + func_name="missing_tool", + tools=[], + ) + + self.assertEqual( + arguments["payload"], + " keep surrounding spaces ", + ) + + +class TestParaImageUrlAlignment(unittest.TestCase): + def test_image_url_string_is_coerced_to_url_object(self): + request = ChatCompletionRequest( + model="test-model", + messages=[ + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": "https://example.test/image.png", + } + ], + } + ], + ) + + image_part = request.messages[0].content[0] + self.assertEqual(image_part.image_url.url, "https://example.test/image.png") + + +class TestParaPayloadTooLargeAlignment(unittest.TestCase): + def test_tokenizer_manager_raises_payload_too_large_for_glm_input_overflow(self): + source = ( + _REPO_ROOT / "python/sglang/srt/managers/tokenizer_manager.py" + ).read_text() + + self.assertIn("class PayloadTooLargeError(Exception):", source) + self.assertIn('getattr(self.server_args, "openai_glm_compat", False)', source) + self.assertIn('"glm" in self.model_path.lower()', source) + self.assertIn("raise PayloadTooLargeError(error_msg)", source) + + def test_serving_base_maps_payload_too_large_to_http_413(self): + source = ( + _REPO_ROOT / "python/sglang/srt/entrypoints/openai/serving_base.py" + ).read_text() + + self.assertIn( + "from sglang.srt.managers.tokenizer_manager import PayloadTooLargeError", + source, + ) + self.assertIn("except PayloadTooLargeError as e:", source) + self.assertIn('err_type="PayloadTooLargeError"', source) + self.assertIn("status_code=413", source) + + +class TestParaStreamingErrorAlignment(unittest.TestCase): + def test_serving_base_builds_http_error_from_first_streaming_chunk(self): + _install_actual_serving_base_stubs() + from sglang.srt.entrypoints.openai.serving_base import OpenAIServingBase + + class ConcreteServingBase(OpenAIServingBase): + def _request_id_prefix(self): + return "test-" + + def _convert_to_internal_request(self, request, raw_request=None): + return request, request + + serving = ConcreteServingBase.__new__(ConcreteServingBase) + + response = serving.create_error_response_from_first_streaming_chunk( + 'data: {"error": {"message": "too large", "type": ' + '"PayloadTooLargeError", "code": 413, "param": "messages"}}\n\n' + ) + + self.assertEqual(response.status_code, 413) + self.assertEqual(response.content["message"], "too large") + self.assertEqual(response.content["type"], "PayloadTooLargeError") + self.assertEqual(response.content["param"], "messages") + + def test_chat_streaming_prefetch_returns_http_error_before_sse(self): + _install_serving_chat_stubs() + from sglang.srt.entrypoints.openai.serving_chat import OpenAIServingChat + + async def error_generator(): + yield ( + 'data: {"error": {"message": "too large", "type": ' + '"PayloadTooLargeError", "code": 413}}\n\n' + ) + + serving = OpenAIServingChat.__new__(OpenAIServingChat) + serving.tokenizer_manager = SimpleNamespace( + server_args=SimpleNamespace(openai_streaming_error_preflight=True), + create_abort_task=lambda _adapted_request: None + ) + serving._generate_chat_stream = ( + lambda adapted_request, request, raw_request: error_generator() + ) + + response = asyncio.run( + serving._handle_streaming_request( + adapted_request=SimpleNamespace(), + request=ChatCompletionRequest( + model="test-model", + messages=[{"role": "user", "content": "hello"}], + stream=True, + ), + raw_request=None, + ) + ) + + self.assertEqual(response.status_code, 413) + self.assertEqual(response.content["message"], "too large") + + +class TestParaUsageAlignment(unittest.TestCase): + def test_usage_processor_aggregates_reasoning_tokens(self): + usage = UsageProcessor.calculate_response_usage( + [ + { + "meta_info": { + "prompt_tokens": 10, + "completion_tokens": 3, + "reasoning_tokens": 2, + } + }, + { + "meta_info": { + "prompt_tokens": 10, + "completion_tokens": 4, + "reasoning_tokens": 5, + } + }, + ], + n_choices=2, + ) + + self.assertEqual(usage.prompt_tokens, 10) + self.assertEqual(usage.completion_tokens, 7) + self.assertEqual(usage.reasoning_tokens, 7) + + def test_usage_processor_reports_glm_completion_token_details(self): + usage = UsageProcessor.calculate_streaming_usage( + prompt_tokens={0: 10, 1: 10}, + completion_tokens={0: 3, 1: 4}, + reasoning_tokens={0: 2, 1: 5}, + cached_tokens={}, + n_choices=2, + use_completion_details=True, + ) + + self.assertEqual(usage.prompt_tokens, 10) + self.assertEqual(usage.completion_tokens, 7) + self.assertIsNotNone(usage.completion_tokens_details) + self.assertEqual(usage.completion_tokens_details.reasoning_tokens, 7) + + def test_chat_serving_passes_glm_completion_detail_flag(self): + source = ( + _REPO_ROOT / "python/sglang/srt/entrypoints/openai/serving_chat.py" + ).read_text() + + self.assertIn("reasoning_tokens = {}", source) + self.assertIn('reasoning_tokens[index] = content["meta_info"].get(', source) + self.assertGreaterEqual(source.count("use_completion_details=self.is_glm"), 3) + + +if __name__ == "__main__": + unittest.main(verbosity=2)