chenchenghao/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

Align OpenAI serving behavior with Para deployments

Browse Source 
Absorb PR 11's final Para compatibility surface as an opt-in OpenAI serving layer rather than hard-coding business defaults into protocol models. The change adds server args for Para chat defaults, Kimi/GLM compatibility, tool-choice normalization, tool-role text flattening, and streaming first-chunk error preflight while preserving default upstream behavior unless explicitly enabled.

Reasoning token usage is also propagated through chat/completion usage paths, with GLM compatibility emitting completion_tokens_details.reasoning_tokens. Low-risk protocol fixes accept string image_url content parts and preserve GLM function-call argument value whitespace.

Constraint: Online Para-compatible deployments require request/response semantics that differ from default OpenAI serving behavior.

Constraint: Current CP/HiCache/bs>1 work must not be coupled to OpenAI serving compatibility changes.

Rejected: Merge PR 11 history directly | intermediate commits briefly hard-code chat max_tokens=32768 before later gating it by server args.

Rejected: Enable Para compatibility by default | would change non-Para OpenAI-compatible deployments.

Confidence: high

Scope-risk: moderate

Directive: Keep Para-specific serving policies behind explicit server args unless the business contract changes globally.

Tested: PYTHONPATH=python:. python -m unittest discover -s test/registered/unit/entrypoints/openai -p 'test_para_serving_protocol.py' -v (19 tests OK)

Tested: python -m py_compile modified OpenAI serving, tokenizer manager, server_args, function-call detector, and test files

Not-tested: Live router/prefill/decode OpenAI serving E2E after enabling Para flags.

Co-authored-by: OmX <omx@oh-my-codex.dev>
This commit is contained in:
laoyao0822
2026-06-11 05:59:08 +08:00
parent 8cfdb0466e
commit 9c8e3e99cb
11 changed files with 1262 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
python/sglang/srt/entrypoints/openai/protocol.py
View File

@@ -128,6 +128,12 @@ class PromptTokensDetails(BaseModel):
cached_tokens: int = 0
class CompletionTokensDetails(BaseModel):
"""Details about completion tokens."""
reasoning_tokens: int = 0
class UsageInfo(BaseModel):
prompt_tokens: int = 0
total_tokens: int = 0
@@ -135,6 +141,7 @@ class UsageInfo(BaseModel):
# Used to return cached tokens info when --enable-cache-report is set
prompt_tokens_details: Optional[PromptTokensDetails] = None
reasoning_tokens: Optional[int] = 0
completion_tokens_details: Optional[CompletionTokensDetails] = None
class StreamOptions(BaseModel):
@@ -426,6 +433,13 @@ class ChatCompletionMessageContentImageURL(BaseModel):
max_dynamic_patch: Optional[int] = None
min_dynamic_patch: Optional[int] = None
@model_validator(mode="before")
@classmethod
def coerce_string(cls, values):
if isinstance(values, str):
return {"url": values}
return values
class ChatCompletionMessageContentVideoURL(BaseModel):
url: str
@@ -615,6 +629,7 @@ class ChatCompletionRequest(BaseModel):
separate_reasoning: bool = True
stream_reasoning: bool = True
chat_template_kwargs: Optional[Dict] = None
thinking: Optional[Dict] = None
# SGLang multimodal tiling controls (extensions)
max_dynamic_patch: Optional[int] = None
@@ -743,13 +758,16 @@ class ChatCompletionRequest(BaseModel):
stop: List[str],
model_generation_config: Dict[str, Any],
tool_call_constraint: Optional[ToolCallConstraint] = None,
fixed_sampling_overrides: Optional[Dict[str, Any]] = None,
) -> Dict[str, Any]:
"""
Convert request to sampling parameters.
Priority: user value > model generation_config > OpenAI defaults
Priority: fixed_sampling_overrides (if any) > user value > model_generation_config > OpenAI defaults
"""
def get_param(param_name: str):
if fixed_sampling_overrides and param_name in fixed_sampling_overrides:
return fixed_sampling_overrides[param_name]
value = getattr(self, param_name)
if value is None:
return model_generation_config.get(
@@ -779,7 +797,11 @@ class ChatCompletionRequest(BaseModel):
"repetition_penalty": get_param("repetition_penalty"),
"regex": self.regex,
"ebnf": self.ebnf,
"n": self.n,
"n": (
fixed_sampling_overrides["n"]
if fixed_sampling_overrides and "n" in fixed_sampling_overrides
else self.n
),
"no_stop_trim": self.no_stop_trim,
"ignore_eos": self.ignore_eos,
"skip_special_tokens": self.skip_special_tokens,

53
python/sglang/srt/entrypoints/openai/serving_base.py
View File

@@ -14,6 +14,7 @@ from fastapi.responses import ORJSONResponse, StreamingResponse
from sglang.srt.entrypoints.openai.encoding_dsv32 import DS32EncodingError
from sglang.srt.entrypoints.openai.protocol import ErrorResponse, OpenAIServingRequest
from sglang.srt.managers.io_struct import EmbeddingReqInput, GenerateReqInput
from sglang.srt.managers.tokenizer_manager import PayloadTooLargeError
from sglang.srt.observability.req_time_stats import monotonic_time
from sglang.srt.server_args import ServerArgs
@@ -135,6 +136,12 @@ class OpenAIServingBase(ABC):
err_type="BadRequest",
status_code=400,
)
except PayloadTooLargeError as e:
return self.create_error_response(
message=str(e),
err_type="PayloadTooLargeError",
status_code=413,
)
except DS32EncodingError as e:
logger.info(f"DS32EncodingError: {e}")
return self.create_error_response(
@@ -242,6 +249,52 @@ class OpenAIServingBase(ABC):
)
return ORJSONResponse(content=error.model_dump(), status_code=status_code)
def create_error_response_from_first_streaming_chunk(
self,
first_chunk: str,
) -> Optional[ORJSONResponse]:
if not isinstance(first_chunk, str):
return None
first_chunk = first_chunk.strip()
if not first_chunk.startswith("data:"):
return None
data = first_chunk[len("data:") :].strip()
if data == "[DONE]":
return None
try:
payload = json.loads(data)
except json.JSONDecodeError:
return None
if not isinstance(payload, dict):
return None
error = payload.get("error")
if not isinstance(error, dict):
return None
status_code = (
error.get("code")
or error.get("status")
or error.get("status_code")
or 500
)
if not isinstance(status_code, int) or not 100 <= status_code <= 599:
status_code = 500
return self.create_error_response(
message=error.get(
"message",
"Streaming request failed before first chunk.",
),
err_type=error.get("type", "InternalServerError"),
status_code=status_code,
param=error.get("param"),
)
def create_streaming_error_response(
self,
message: str,

169
python/sglang/srt/entrypoints/openai/serving_chat.py
View File

@@ -48,6 +48,7 @@ from sglang.srt.function_call.function_call_parser import FunctionCallParser
from sglang.srt.function_call.json_array_parser import JsonArrayParser
from sglang.srt.function_call.utils import get_json_schema_constraint
from sglang.srt.managers.io_struct import GenerateReqInput
from sglang.srt.managers.tokenizer_manager import PayloadTooLargeError
from sglang.srt.parser.conversation import generate_chat_conv
from sglang.srt.parser.jinja_template_utils import process_content_for_template_format
from sglang.srt.parser.reasoning_parser import ReasoningParser
@@ -133,6 +134,16 @@ class OpenAIServingChat(OpenAIServingBase):
and self.tokenizer_manager.model_config.hf_config.model_type == "gpt_oss"
)
# Model-specific Para compatibility is controlled by server args so local
# deployments can opt out without editing OpenAI protocol models.
model_path = self.tokenizer_manager.server_args.model_path.lower()
self.is_kimi = self._get_server_arg("openai_kimi_compat", False) and (
"kimi" in model_path
)
self.is_glm = self._get_server_arg("openai_glm_compat", False) and (
"glm" in model_path
)
self.use_dpsk_v32_encoding = self._use_dpsk_v32_encoding()
def _handle_last_assistant_message(
@@ -327,8 +338,42 @@ class OpenAIServingChat(OpenAIServingBase):
def _request_id_prefix(self) -> str:
return "chatcmpl-"
def _get_server_arg(self, name: str, default: Any) -> Any:
server_args = getattr(self.tokenizer_manager, "server_args", None)
return getattr(server_args, name, default)
def _apply_openai_serving_defaults(
self, request: ChatCompletionRequest
) -> None:
if request.max_completion_tokens is not None or request.max_tokens is not None:
return
default_max_tokens = self._get_server_arg(
"openai_chat_default_max_tokens", 0
)
if default_max_tokens and default_max_tokens > 0:
request.max_tokens = default_max_tokens
def _validate_request(self, request: ChatCompletionRequest) -> Optional[str]:
"""Validate that the input is valid."""
self._apply_openai_serving_defaults(request)
if (
self._get_server_arg("openai_force_tool_choice_auto", False)
and request.tool_choice != "auto"
):
request.tool_choice = "auto"
if (
getattr(self, "is_glm", False)
and isinstance(request.tool_choice, str)
and request.tool_choice.lower() == "required"
):
logger.warning(
"tool_choice='required' is being downgraded to 'auto' for GLM model."
)
request.tool_choice = "auto"
if not request.messages:
return "Messages cannot be empty."
@@ -373,13 +418,50 @@ class OpenAIServingChat(OpenAIServingBase):
if schema is None:
return "schema_ is required for json_schema response format request."
if getattr(self, "is_kimi", False):
is_think_mode = not (
request.chat_template_kwargs
and request.chat_template_kwargs.get("thinking") is False
)
expected_params = self._get_kimi_fixed_params(is_think_mode)
for param, expected_value in expected_params.items():
user_value = getattr(request, param)
if user_value is not None and abs(user_value - expected_value) >= 1e-3:
return (
f"Parameter '{param}' cannot be overridden. "
f"Expected: {expected_value}, Got: {user_value}"
)
user_temperature = getattr(request, "temperature")
if user_temperature is not None and (
user_temperature < 0.0 or user_temperature > 1.0
):
return (
"Parameter `temperature` must be in [0, 1.0]. "
f"Got: {user_temperature}"
)
return None
@staticmethod
def _get_kimi_fixed_params(is_think_mode: bool) -> Dict[str, float]:
"""Return Kimi's fixed sampling parameters based on thinking mode."""
return {
# Keep Para behavior: temperature is validated as a range, but not forced.
# "temperature": 1.0 if is_think_mode else 0.6,
"top_p": 0.95,
"presence_penalty": 0.0,
"frequency_penalty": 0.0,
"n": 1,
}
def _convert_to_internal_request(
self,
request: ChatCompletionRequest,
raw_request: Request = None,
) -> tuple[GenerateReqInput, ChatCompletionRequest]:
self._apply_openai_serving_defaults(request)
reasoning_effort = (
request.chat_template_kwargs.pop("reasoning_effort", None)
if request.chat_template_kwargs
@@ -404,10 +486,18 @@ class OpenAIServingChat(OpenAIServingBase):
raise
# Build sampling parameters
fixed_overrides = None
if getattr(self, "is_kimi", False):
is_think_mode = not (
request.chat_template_kwargs
and request.chat_template_kwargs.get("thinking") is False
)
fixed_overrides = self._get_kimi_fixed_params(is_think_mode)
sampling_params = request.to_sampling_params(
stop=processed_messages.stop,
model_generation_config=self.default_sampling_params,
tool_call_constraint=processed_messages.tool_call_constraint,
fixed_sampling_overrides=fixed_overrides,
)
# Handle single vs multiple requests
@@ -530,6 +620,13 @@ class OpenAIServingChat(OpenAIServingBase):
template_content_format = self.template_manager.jinja_template_content_format
if getattr(self, "is_kimi", False) and request.thinking is not None:
if request.chat_template_kwargs is None:
request.chat_template_kwargs = {}
request.chat_template_kwargs["thinking"] = (
request.thinking.get("type") != "disabled"
)
if self.use_dpsk_v32_encoding:
thinking_mode = (
"thinking"
@@ -587,6 +684,30 @@ class OpenAIServingChat(OpenAIServingBase):
modalities,
)
# Normalize tool role content: OpenAI clients may send content as a
# list of content parts, but most chat templates expect a plain
# string for tool messages. Only flatten pure text parts; preserve
# lists that carry tool-semantic fields for templates that iterate
# over those structures.
if (
self._get_server_arg(
"openai_flatten_tool_role_text_content", False
)
and processed_msg["role"] == "tool"
and isinstance(processed_msg.get("content"), list)
):
parts = processed_msg["content"]
is_openai_text_parts = all(
(isinstance(part, dict) and part.get("type") == "text")
or isinstance(part, str)
for part in parts
)
if is_openai_text_parts:
processed_msg["content"] = "\n".join(
part.get("text", "") if isinstance(part, dict) else part
for part in parts
)
# per the Transformers docs & maintainers, tool call arguments in
# assistant-role messages with tool_calls need to be dicts not JSON str -
# this is how tool-use chat templates will expect them moving forwards
@@ -745,8 +866,37 @@ class OpenAIServingChat(OpenAIServingBase):
raw_request: Request,
) -> StreamingResponse:
"""Handle streaming chat completion request"""
if not self._get_server_arg("openai_streaming_error_preflight", False):
return StreamingResponse(
self._generate_chat_stream(adapted_request, request, raw_request),
media_type="text/event-stream",
background=self.tokenizer_manager.create_abort_task(adapted_request),
)
generator = self._generate_chat_stream(adapted_request, request, raw_request)
try:
first_chunk = await generator.__anext__()
except PayloadTooLargeError as e:
return self.create_error_response(
str(e), status_code=413, err_type="PayloadTooLargeError"
)
except ValueError as e:
return self.create_error_response(str(e))
first_chunk_error_response = self.create_error_response_from_first_streaming_chunk(
first_chunk
)
if first_chunk_error_response is not None:
return first_chunk_error_response
async def prepend_first_chunk():
yield first_chunk
async for chunk in generator:
yield chunk
return StreamingResponse(
self._generate_chat_stream(adapted_request, request, raw_request),
prepend_first_chunk(),
media_type="text/event-stream",
background=self.tokenizer_manager.create_abort_task(adapted_request),
)
@@ -772,6 +922,7 @@ class OpenAIServingChat(OpenAIServingBase):
# Usage tracking
prompt_tokens = {}
completion_tokens = {}
reasoning_tokens = {}
cached_tokens = {}
hidden_states = {}
routed_experts = {}
@@ -786,6 +937,9 @@ class OpenAIServingChat(OpenAIServingBase):
completion_tokens[index] = content["meta_info"].get(
"completion_tokens", 0
)
reasoning_tokens[index] = content["meta_info"].get(
"reasoning_tokens", 0
)
cached_tokens[index] = content["meta_info"].get("cached_tokens", 0)
hidden_states[index] = content["meta_info"].get("hidden_states", None)
routed_experts[index] = content["meta_info"].get("routed_experts", None)
@@ -874,6 +1028,8 @@ class OpenAIServingChat(OpenAIServingBase):
chunk.usage = UsageProcessor.calculate_token_usage(
prompt_tokens=prompt_tokens.get(index, 0),
completion_tokens=completion_tokens.get(index, 0),
reasoning_tokens=reasoning_tokens.get(index, 0),
use_completion_details=self.is_glm,
)
yield f"data: {chunk.model_dump_json()}\n\n"
@@ -929,6 +1085,8 @@ class OpenAIServingChat(OpenAIServingBase):
chunk.usage = UsageProcessor.calculate_token_usage(
prompt_tokens=prompt_tokens.get(index, 0),
completion_tokens=completion_tokens.get(index, 0),
reasoning_tokens=reasoning_tokens.get(index, 0),
use_completion_details=self.is_glm,
)
yield f"data: {chunk.model_dump_json()}\n\n"
@@ -1012,6 +1170,8 @@ class OpenAIServingChat(OpenAIServingBase):
cached_tokens,
n_choices=request.n,
enable_cache_report=self.tokenizer_manager.server_args.enable_cache_report,
reasoning_tokens=reasoning_tokens,
use_completion_details=self.is_glm,
)
usage_chunk = ChatCompletionStreamResponse(
id=content["meta_info"]["id"],
@@ -1151,6 +1311,7 @@ class OpenAIServingChat(OpenAIServingBase):
ret,
n_choices=request.n,
enable_cache_report=self.tokenizer_manager.server_args.enable_cache_report,
use_completion_details=self.is_glm,
)
return ChatCompletionResponse(
@@ -1470,9 +1631,12 @@ class OpenAIServingChat(OpenAIServingBase):
if request.stream_options and request.stream_options.continuous_usage_stats:
prompt_tokens = content["meta_info"].get("prompt_tokens", 0)
completion_tokens = content["meta_info"].get("completion_tokens", 0)
reasoning_tokens = content["meta_info"].get("reasoning_tokens", 0)
chunk.usage = UsageProcessor.calculate_token_usage(
prompt_tokens=prompt_tokens,
completion_tokens=completion_tokens,
reasoning_tokens=reasoning_tokens,
use_completion_details=self.is_glm,
)
yield f"data: {chunk.model_dump_json()}\n\n"
@@ -1521,9 +1685,12 @@ class OpenAIServingChat(OpenAIServingBase):
if request.stream_options and request.stream_options.continuous_usage_stats:
prompt_tokens = content["meta_info"].get("prompt_tokens", 0)
completion_tokens = content["meta_info"].get("completion_tokens", 0)
reasoning_tokens = content["meta_info"].get("reasoning_tokens", 0)
chunk.usage = UsageProcessor.calculate_token_usage(
prompt_tokens=prompt_tokens,
completion_tokens=completion_tokens,
reasoning_tokens=reasoning_tokens,
use_completion_details=self.is_glm,
)
yield f"data: {chunk.model_dump_json()}\n\n"

6
python/sglang/srt/entrypoints/openai/serving_completions.py
View File

@@ -204,6 +204,7 @@ class OpenAIServingCompletion(OpenAIServingBase):
# Usage tracking
prompt_tokens = {}
completion_tokens = {}
reasoning_tokens = {}
cached_tokens = {}
hidden_states = {}
routed_experts = {}
@@ -219,6 +220,9 @@ class OpenAIServingCompletion(OpenAIServingBase):
completion_tokens[index] = content["meta_info"].get(
"completion_tokens", 0
)
reasoning_tokens[index] = content["meta_info"].get(
"reasoning_tokens", 0
)
cached_tokens[index] = content["meta_info"].get("cached_tokens", 0)
hidden_states[index] = content["meta_info"].get("hidden_states", None)
routed_experts[index] = content["meta_info"].get("routed_experts", None)
@@ -312,6 +316,7 @@ class OpenAIServingCompletion(OpenAIServingBase):
chunk.usage = UsageProcessor.calculate_token_usage(
prompt_tokens=prompt_tokens.get(index, 0),
completion_tokens=completion_tokens.get(index, 0),
reasoning_tokens=reasoning_tokens.get(index, 0),
)
yield f"data: {chunk.model_dump_json()}\n\n"
@@ -364,6 +369,7 @@ class OpenAIServingCompletion(OpenAIServingBase):
cached_tokens,
n_choices=request.n,
enable_cache_report=self.tokenizer_manager.server_args.enable_cache_report,
reasoning_tokens=reasoning_tokens,
)
final_usage_chunk = CompletionStreamResponse(
id=content["meta_info"]["id"],

35
python/sglang/srt/entrypoints/openai/usage_processor.py
View File

@@ -2,7 +2,11 @@ from __future__ import annotations
from typing import Any, Dict, List, Mapping, Optional, final
from sglang.srt.entrypoints.openai.protocol import PromptTokensDetails, UsageInfo
from sglang.srt.entrypoints.openai.protocol import (
CompletionTokensDetails,
PromptTokensDetails,
UsageInfo,
)
@final
@@ -19,10 +23,14 @@ class UsageProcessor:
responses: List[Dict[str, Any]],
n_choices: int = 1,
enable_cache_report: bool = False,
use_completion_details: bool = False,
) -> UsageInfo:
completion_tokens = sum(
r["meta_info"].get("completion_tokens", 0) for r in responses
)
reasoning_tokens = sum(
r["meta_info"].get("reasoning_tokens", 0) for r in responses
)
prompt_tokens = sum(
responses[i]["meta_info"].get("prompt_tokens", 0)
@@ -41,6 +49,8 @@ class UsageProcessor:
prompt_tokens=prompt_tokens,
completion_tokens=completion_tokens,
cached_tokens=cached_details,
reasoning_tokens=reasoning_tokens,
use_completion_details=use_completion_details,
)
@staticmethod
@@ -50,12 +60,17 @@ class UsageProcessor:
cached_tokens: Mapping[int, int],
n_choices: int,
enable_cache_report: bool = False,
reasoning_tokens: Optional[Mapping[int, int]] = None,
use_completion_details: bool = False,
) -> UsageInfo:
# index % n_choices == 0 marks the first choice of a prompt
total_prompt_tokens = sum(
tok for idx, tok in prompt_tokens.items() if idx % n_choices == 0
)
total_completion_tokens = sum(completion_tokens.values())
total_reasoning_tokens = (
sum(reasoning_tokens.values()) if reasoning_tokens is not None else 0
)
cached_details = (
UsageProcessor._details_if_cached(
@@ -69,6 +84,8 @@ class UsageProcessor:
prompt_tokens=total_prompt_tokens,
completion_tokens=total_completion_tokens,
cached_tokens=cached_details,
reasoning_tokens=total_reasoning_tokens,
use_completion_details=use_completion_details,
)
@staticmethod
@@ -76,11 +93,27 @@ class UsageProcessor:
prompt_tokens: int,
completion_tokens: int,
cached_tokens: Optional[PromptTokensDetails] = None,
reasoning_tokens: Optional[int] = 0,
use_completion_details: bool = False,
) -> UsageInfo:
"""Calculate token usage information"""
if use_completion_details:
return UsageInfo(
prompt_tokens=prompt_tokens,
completion_tokens=completion_tokens,
total_tokens=prompt_tokens + completion_tokens,
prompt_tokens_details=cached_tokens,
completion_tokens_details=(
CompletionTokensDetails(reasoning_tokens=reasoning_tokens)
if reasoning_tokens
else None
),
)
return UsageInfo(
prompt_tokens=prompt_tokens,
completion_tokens=completion_tokens,
total_tokens=prompt_tokens + completion_tokens,
prompt_tokens_details=cached_tokens,
reasoning_tokens=reasoning_tokens,
)

1
python/sglang/srt/function_call/glm47_moe_detector.py
View File

@@ -759,7 +759,6 @@ class Glm47MoeDetector(BaseFormatDetector):
arguments = {}
for arg_key, arg_value in pairs:
arg_key = arg_key.strip()
arg_value = arg_value.strip()
arg_type = get_argument_type(func_name, arg_key, tools)
parsed_value, is_good_json = parse_arguments(arg_value, arg_type)

1
python/sglang/srt/function_call/glm4_moe_detector.py
View File

@@ -613,7 +613,6 @@ class Glm4MoeDetector(BaseFormatDetector):
arguments = {}
for arg_key, arg_value in pairs:
arg_key = arg_key.strip()
arg_value = arg_value.strip()
arg_type = get_argument_type(func_name, arg_key, tools)
parsed_value, is_good_json = parse_arguments(arg_value, arg_type)

18
python/sglang/srt/managers/tokenizer_manager.py
View File

@@ -121,6 +121,11 @@ asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
_REQUEST_STATE_WAIT_TIMEOUT = envs.SGLANG_REQUEST_STATE_WAIT_TIMEOUT.get()
class PayloadTooLargeError(Exception):
"""Exception raised when a request payload exceeds the model context length."""
logger = logging.getLogger(__name__)
@@ -806,10 +811,16 @@ class TokenizerManager(TokenizerCommunicatorMixin, TokenizerManagerMultiItemMixi
del input_ids[_max_req_len:]
input_token_num = len(input_ids)
else:
raise ValueError(
error_msg = (
f"The input ({input_token_num} tokens) is longer than the "
f"model's context length ({self.context_len} tokens)."
)
if (
getattr(self.server_args, "openai_glm_compat", False)
and "glm" in self.model_path.lower()
):
raise PayloadTooLargeError(error_msg)
raise ValueError(error_msg)
# Validate total tokens (input + max_new_tokens)
max_new_tokens = obj.sampling_params.get("max_new_tokens")
@@ -836,6 +847,11 @@ class TokenizerManager(TokenizerCommunicatorMixin, TokenizerManagerMultiItemMixi
f"{max_new_tokens} tokens for the completion. Please reduce the number "
f"of tokens in the input messages or the completion to fit within the limit."
)
if (
getattr(self.server_args, "openai_glm_compat", False)
and "glm" in self.model_path.lower()
):
raise PayloadTooLargeError(error_msg)
raise ValueError(error_msg)
# Validate embedding requests

66
python/sglang/srt/server_args.py
View File

@@ -454,6 +454,12 @@ class ServerArgs:
tool_call_parser: Optional[str] = None
tool_server: Optional[str] = None
sampling_defaults: str = "model"
openai_chat_default_max_tokens: int = 0
openai_force_tool_choice_auto: bool = False
openai_kimi_compat: bool = False
openai_glm_compat: bool = False
openai_flatten_tool_role_text_content: bool = False
openai_streaming_error_preflight: bool = False
# Data parallelism
dp_size: int = 1
@@ -4688,6 +4694,66 @@ class ServerArgs:
"'model' uses the model's generation_config.json to get the recommended "
"sampling parameters if available. Default is 'model'.",
)
parser.add_argument(
"--openai-chat-default-max-tokens",
type=int,
default=ServerArgs.openai_chat_default_max_tokens,
help=(
"Default max_tokens applied to OpenAI chat requests that omit both "
"max_tokens and max_completion_tokens. Set to 0 or a negative value "
"to preserve the upstream unset default."
),
)
parser.add_argument(
"--openai-force-tool-choice-auto",
action="store_true",
dest="openai_force_tool_choice_auto",
default=ServerArgs.openai_force_tool_choice_auto,
help=(
"Enable Para-compatible normalization that serves explicit "
"non-auto chat tool_choice values as 'auto'."
),
)
parser.add_argument(
"--openai-kimi-compat",
action="store_true",
dest="openai_kimi_compat",
default=ServerArgs.openai_kimi_compat,
help=(
"Enable Para-compatible Kimi handling, including thinking field "
"mapping and fixed Kimi sampling parameter enforcement."
),
)
parser.add_argument(
"--openai-glm-compat",
action="store_true",
dest="openai_glm_compat",
default=ServerArgs.openai_glm_compat,
help=(
"Enable Para-compatible GLM handling, including required tool_choice "
"downgrade, 413 context overflow mapping, and GLM usage details."
),
)
parser.add_argument(
"--openai-flatten-tool-role-text-content",
action="store_true",
dest="openai_flatten_tool_role_text_content",
default=ServerArgs.openai_flatten_tool_role_text_content,
help=(
"Enable Para-compatible flattening of pure text content-part lists "
"on OpenAI tool-role messages before chat-template rendering."
),
)
parser.add_argument(
"--openai-streaming-error-preflight",
action="store_true",
dest="openai_streaming_error_preflight",
default=ServerArgs.openai_streaming_error_preflight,
help=(
"Enable prefetching the first OpenAI streaming chunk to convert "
"pre-stream errors into normal HTTP error responses."
),
)
# Data parallelism
parser.add_argument(