import asyncio import sys import types import unittest from pathlib import Path from types import SimpleNamespace from pydantic import BaseModel def _install_openai_response_stubs(): responses_mod = types.ModuleType("openai.types.responses") response_mod = types.ModuleType("openai.types.responses.response") tool_mod = types.ModuleType("openai.types.responses.tool") class _OpenAIStubModel(BaseModel): pass for name in ( "ResponseFunctionToolCall", "ResponseInputItemParam", "ResponseOutputItem", "ResponseOutputMessage", "ResponseOutputText", "ResponseReasoningItem", ): setattr(responses_mod, name, type(name, (_OpenAIStubModel,), {})) response_mod.ToolChoice = type("ToolChoice", (_OpenAIStubModel,), {}) tool_mod.Tool = type("Tool", (_OpenAIStubModel,), {}) sys.modules.setdefault("openai", types.ModuleType("openai")) sys.modules.setdefault("openai.types", types.ModuleType("openai.types")) sys.modules["openai.types.responses"] = responses_mod sys.modules["openai.types.responses.response"] = response_mod sys.modules["openai.types.responses.tool"] = tool_mod _install_openai_response_stubs() from sglang.srt.entrypoints.openai.protocol import ChatCompletionRequest from sglang.srt.entrypoints.openai.usage_processor import UsageProcessor from sglang.test.ci.ci_register import register_cpu_ci register_cpu_ci(est_time=1, suite="stage-a-test-cpu") _REPO_ROOT = Path(__file__).resolve().parents[5] class TestParaChatDefaults(unittest.TestCase): def test_chat_protocol_leaves_omitted_max_tokens_unset(self): request = ChatCompletionRequest( model="test-model", messages=[{"role": "user", "content": "hello"}], ) self.assertIsNone(request.max_tokens) def test_serving_applies_configured_para_default_max_tokens(self): _install_serving_chat_stubs() from sglang.srt.entrypoints.openai.serving_chat import OpenAIServingChat request = ChatCompletionRequest( model="test-model", messages=[{"role": "user", "content": "hello"}], ) serving = OpenAIServingChat.__new__(OpenAIServingChat) serving.tokenizer_manager = SimpleNamespace( server_args=SimpleNamespace(openai_chat_default_max_tokens=32768) ) serving._apply_openai_serving_defaults(request) self.assertEqual(request.max_tokens, 32768) sampling_params = request.to_sampling_params(stop=[], model_generation_config={}) self.assertEqual(sampling_params["max_new_tokens"], 32768) def test_serving_can_disable_para_default_max_tokens(self): _install_serving_chat_stubs() from sglang.srt.entrypoints.openai.serving_chat import OpenAIServingChat request = ChatCompletionRequest( model="test-model", messages=[{"role": "user", "content": "hello"}], ) serving = OpenAIServingChat.__new__(OpenAIServingChat) serving.tokenizer_manager = SimpleNamespace( server_args=SimpleNamespace(openai_chat_default_max_tokens=0) ) serving._apply_openai_serving_defaults(request) self.assertIsNone(request.max_tokens) def _install_serving_chat_stubs(): """Install lightweight stubs for optional serving dependencies. The Para alignment tests run on macOS in a small uv virtualenv. They exercise request/serving glue without requiring the full SGLang server dependency set. """ import json orjson_mod = types.ModuleType("orjson") orjson_mod.loads = json.loads orjson_mod.dumps = lambda obj, **_: json.dumps(obj).encode() sys.modules["orjson"] = orjson_mod fastapi_mod = types.ModuleType("fastapi") fastapi_mod.Request = type("Request", (), {}) fastapi_mod.HTTPException = type("HTTPException", (Exception,), {}) responses_mod = types.ModuleType("fastapi.responses") class ORJSONResponse: def __init__(self, content=None, status_code=200, **kwargs): self.content = content self.status_code = status_code self.kwargs = kwargs class StreamingResponse: def __init__(self, content=None, media_type=None, background=None, **kwargs): self.content = content self.media_type = media_type self.background = background self.kwargs = kwargs responses_mod.ORJSONResponse = ORJSONResponse responses_mod.StreamingResponse = StreamingResponse sys.modules["fastapi"] = fastapi_mod sys.modules["fastapi.responses"] = responses_mod jsonschema_mod = types.ModuleType("jsonschema") class Draft202012Validator: @staticmethod def check_schema(_schema): return None jsonschema_mod.Draft202012Validator = Draft202012Validator jsonschema_mod.SchemaError = type("SchemaError", (Exception,), {}) sys.modules["jsonschema"] = jsonschema_mod serving_base_mod = types.ModuleType("sglang.srt.entrypoints.openai.serving_base") class OpenAIServingBase: def __init__(self, tokenizer_manager): self.tokenizer_manager = tokenizer_manager def create_error_response( self, message, err_type="BadRequestError", status_code=400, param=None ): return ORJSONResponse( content={ "message": message, "type": err_type, "param": param, "code": status_code, }, status_code=status_code, ) def create_streaming_error_response(self, message): return json.dumps({"error": {"message": message}}) def create_error_response_from_first_streaming_chunk(self, first_chunk): if not isinstance(first_chunk, str): return None first_chunk = first_chunk.strip() if not first_chunk.startswith("data:"): return None data = first_chunk[len("data:") :].strip() if data == "[DONE]": return None try: payload = json.loads(data) except json.JSONDecodeError: return None error = payload.get("error") if isinstance(payload, dict) else None if not isinstance(error, dict): return None status_code = ( error.get("code") or error.get("status") or error.get("status_code") or 500 ) return self.create_error_response( message=error.get( "message", "Streaming request failed before first chunk." ), err_type=error.get("type", "InternalServerError"), status_code=status_code if isinstance(status_code, int) else 500, param=error.get("param"), ) def extract_custom_labels(self, _raw_request): return None def extract_routed_dp_rank_from_header(self, _raw_request, routed_dp_rank): return routed_dp_rank serving_base_mod.OpenAIServingBase = OpenAIServingBase sys.modules["sglang.srt.entrypoints.openai.serving_base"] = serving_base_mod io_struct_mod = types.ModuleType("sglang.srt.managers.io_struct") class GenerateReqInput: def __init__(self, **kwargs): self.__dict__.update(kwargs) io_struct_mod.GenerateReqInput = GenerateReqInput io_struct_mod.EmbeddingReqInput = type("EmbeddingReqInput", (), {}) sys.modules["sglang.srt.managers.io_struct"] = io_struct_mod tokenizer_manager_mod = types.ModuleType("sglang.srt.managers.tokenizer_manager") tokenizer_manager_mod.PayloadTooLargeError = type( "PayloadTooLargeError", (Exception,), {} ) sys.modules["sglang.srt.managers.tokenizer_manager"] = tokenizer_manager_mod encoding_mod = types.ModuleType("sglang.srt.entrypoints.openai.encoding_dsv32") encoding_mod.encode_messages = lambda messages, thinking_mode="chat": str( (messages, thinking_mode) ) encoding_mod.DS32EncodingError = type("DS32EncodingError", (Exception,), {}) sys.modules["sglang.srt.entrypoints.openai.encoding_dsv32"] = encoding_mod utils_mod = types.ModuleType("sglang.srt.entrypoints.openai.utils") utils_mod.process_cached_tokens_details_from_ret = lambda *_args, **_kwargs: None utils_mod.process_hidden_states_from_ret = lambda *_args, **_kwargs: None utils_mod.process_routed_experts_from_ret = lambda *_args, **_kwargs: None utils_mod.to_openai_style_logprobs = lambda *_args, **_kwargs: None sys.modules["sglang.srt.entrypoints.openai.utils"] = utils_mod core_types_mod = types.ModuleType("sglang.srt.function_call.core_types") core_types_mod.ToolCallItem = type("ToolCallItem", (), {}) sys.modules["sglang.srt.function_call.core_types"] = core_types_mod function_call_parser_mod = types.ModuleType( "sglang.srt.function_call.function_call_parser" ) function_call_parser_mod.FunctionCallParser = type("FunctionCallParser", (), {}) sys.modules[ "sglang.srt.function_call.function_call_parser" ] = function_call_parser_mod json_array_parser_mod = types.ModuleType( "sglang.srt.function_call.json_array_parser" ) json_array_parser_mod.JsonArrayParser = type("JsonArrayParser", (), {}) sys.modules["sglang.srt.function_call.json_array_parser"] = json_array_parser_mod function_utils_mod = types.ModuleType("sglang.srt.function_call.utils") function_utils_mod.get_json_schema_constraint = lambda *_args, **_kwargs: None sys.modules["sglang.srt.function_call.utils"] = function_utils_mod conversation_mod = types.ModuleType("sglang.srt.parser.conversation") conversation_mod.generate_chat_conv = lambda *_args, **_kwargs: None sys.modules["sglang.srt.parser.conversation"] = conversation_mod jinja_utils_mod = types.ModuleType("sglang.srt.parser.jinja_template_utils") jinja_utils_mod.process_content_for_template_format = ( lambda msg, *_args, **_kwargs: msg ) sys.modules["sglang.srt.parser.jinja_template_utils"] = jinja_utils_mod reasoning_mod = types.ModuleType("sglang.srt.parser.reasoning_parser") reasoning_mod.ReasoningParser = type("ReasoningParser", (), {}) sys.modules["sglang.srt.parser.reasoning_parser"] = reasoning_mod def _install_actual_serving_base_stubs(): _install_serving_chat_stubs() req_time_stats_mod = types.ModuleType( "sglang.srt.observability.req_time_stats" ) req_time_stats_mod.monotonic_time = lambda: 0.0 sys.modules["sglang.srt.observability.req_time_stats"] = req_time_stats_mod server_args_mod = types.ModuleType("sglang.srt.server_args") server_args_mod.ServerArgs = type("ServerArgs", (), {}) sys.modules["sglang.srt.server_args"] = server_args_mod sys.modules.pop("sglang.srt.entrypoints.openai.serving_base", None) def _install_function_call_stubs(): import json from dataclasses import dataclass orjson_mod = types.ModuleType("orjson") orjson_mod.loads = json.loads orjson_mod.dumps = lambda obj, **_: json.dumps(obj).encode() sys.modules["orjson"] = orjson_mod partial_json_mod = types.ModuleType("partial_json_parser") partial_json_mod.loads = lambda data, _flags=None: json.loads(data) sys.modules["partial_json_parser"] = partial_json_mod exceptions_mod = types.ModuleType("partial_json_parser.core.exceptions") exceptions_mod.MalformedJSON = type("MalformedJSON", (Exception,), {}) sys.modules["partial_json_parser.core.exceptions"] = exceptions_mod options_mod = types.ModuleType("partial_json_parser.core.options") class Allow: STR = 1 OBJ = 2 ARR = 4 ALL = STR | OBJ | ARR options_mod.Allow = Allow sys.modules["partial_json_parser.core.options"] = options_mod core_types_mod = types.ModuleType("sglang.srt.function_call.core_types") class ToolCallItem(BaseModel): tool_index: int name: str | None = None parameters: str class StreamingParseResult(BaseModel): normal_text: str = "" calls: list[ToolCallItem] = [] @dataclass class StructureInfo: begin: str end: str trigger: str core_types_mod.ToolCallItem = ToolCallItem core_types_mod.StreamingParseResult = StreamingParseResult core_types_mod.StructureInfo = StructureInfo core_types_mod._GetInfoFunc = object sys.modules["sglang.srt.function_call.core_types"] = core_types_mod function_utils_mod = types.ModuleType("sglang.srt.function_call.utils") function_utils_mod._find_common_prefix = lambda left, right: "" function_utils_mod._is_complete_json = lambda data: True function_utils_mod._partial_json_loads = ( lambda data, _flags=None: (json.loads(data), len(data)) ) function_utils_mod.infer_type_from_json_schema = ( lambda schema: schema.get("type") if isinstance(schema, dict) else None ) function_utils_mod.get_json_schema_constraint = lambda *_args, **_kwargs: None sys.modules["sglang.srt.function_call.utils"] = function_utils_mod class TestParaKimiAlignment(unittest.TestCase): def test_chat_accepts_thinking_and_fixed_sampling_overrides_win(self): request = ChatCompletionRequest( model="kimi-test-model", messages=[{"role": "user", "content": "hello"}], thinking={"type": "disabled"}, top_p=0.1, n=2, ) self.assertEqual(request.thinking, {"type": "disabled"}) sampling_params = request.to_sampling_params( stop=[], model_generation_config={"top_p": 0.7}, fixed_sampling_overrides={"top_p": 0.95, "n": 1}, ) self.assertEqual(sampling_params["top_p"], 0.95) self.assertEqual(sampling_params["n"], 1) def test_kimi_thinking_maps_to_chat_template_kwargs(self): _install_serving_chat_stubs() from sglang.srt.entrypoints.openai.serving_chat import OpenAIServingChat captured = {} class FakeTokenizer: def apply_chat_template(self, messages, **kwargs): captured["messages"] = messages captured["kwargs"] = kwargs return [1, 2, 3] serving = OpenAIServingChat.__new__(OpenAIServingChat) serving.is_kimi = True serving.use_dpsk_v32_encoding = False serving.template_manager = SimpleNamespace( jinja_template_content_format="openai" ) serving.tokenizer_manager = SimpleNamespace(tokenizer=FakeTokenizer()) request = ChatCompletionRequest( model="kimi-test-model", messages=[{"role": "user", "content": "hello"}], thinking={"type": "disabled"}, ) serving._apply_jinja_template(request, tools=None, is_multimodal=False) self.assertEqual(request.chat_template_kwargs, {"thinking": False}) self.assertEqual(captured["kwargs"]["thinking"], False) class TestParaGlmAlignment(unittest.TestCase): def test_serving_chat_detects_glm_model_path(self): _install_serving_chat_stubs() from sglang.srt.entrypoints.openai.serving_chat import OpenAIServingChat tokenizer_manager = SimpleNamespace( server_args=SimpleNamespace( tool_call_parser=None, reasoning_parser=None, model_path="/models/GLM-4.5", openai_glm_compat=True, openai_kimi_compat=False, ), model_config=SimpleNamespace( get_default_sampling_params=lambda: {}, hf_config=SimpleNamespace(model_type="glm", architectures=[]), ), tokenizer=SimpleNamespace(chat_template="template"), ) serving = OpenAIServingChat( tokenizer_manager, template_manager=SimpleNamespace(jinja_template_content_format="openai"), ) self.assertTrue(serving.is_glm) self.assertFalse(serving.is_kimi) def test_glm_required_tool_choice_is_downgraded_to_auto(self): _install_serving_chat_stubs() from sglang.srt.entrypoints.openai.serving_chat import OpenAIServingChat serving = OpenAIServingChat.__new__(OpenAIServingChat) serving.is_glm = True serving.is_kimi = False serving.tokenizer_manager = SimpleNamespace( server_args=SimpleNamespace( context_length=65536, allow_auto_truncate=False, openai_force_tool_choice_auto=False, openai_chat_default_max_tokens=32768, ) ) request = ChatCompletionRequest( model="glm-test-model", messages=[{"role": "user", "content": "hello"}], tools=[ { "type": "function", "function": {"name": "lookup", "parameters": {"type": "object"}}, } ], tool_choice="required", ) error = serving._validate_request(request) self.assertIsNone(error) self.assertEqual(request.tool_choice, "auto") class TestParaToolChoiceAlignment(unittest.TestCase): def test_protocol_preserves_explicit_non_auto_tool_choice(self): required_request = ChatCompletionRequest( model="test-model", messages=[{"role": "user", "content": "hello"}], tools=[ { "type": "function", "function": {"name": "lookup", "parameters": {"type": "object"}}, } ], tool_choice="required", ) none_request = ChatCompletionRequest( model="test-model", messages=[{"role": "user", "content": "hello"}], tool_choice="none", ) self.assertEqual(required_request.tool_choice, "required") self.assertEqual(none_request.tool_choice, "none") def test_serving_arg_coerces_explicit_non_auto_tool_choice_to_auto(self): _install_serving_chat_stubs() from sglang.srt.entrypoints.openai.serving_chat import OpenAIServingChat serving = OpenAIServingChat.__new__(OpenAIServingChat) serving.is_glm = False serving.is_kimi = False serving.tokenizer_manager = SimpleNamespace( server_args=SimpleNamespace( context_length=65536, allow_auto_truncate=False, openai_force_tool_choice_auto=True, openai_chat_default_max_tokens=32768, ) ) request = ChatCompletionRequest( model="test-model", messages=[{"role": "user", "content": "hello"}], tool_choice="none", ) error = serving._validate_request(request) self.assertIsNone(error) self.assertEqual(request.tool_choice, "auto") class TestParaToolCallAlignment(unittest.TestCase): def test_tool_role_text_content_parts_are_flattened_for_chat_templates(self): _install_serving_chat_stubs() from sglang.srt.entrypoints.openai.serving_chat import OpenAIServingChat captured = {} class FakeTokenizer: def apply_chat_template(self, messages, **kwargs): captured["messages"] = messages captured["kwargs"] = kwargs return [1, 2, 3] serving = OpenAIServingChat.__new__(OpenAIServingChat) serving.is_kimi = False serving.use_dpsk_v32_encoding = False serving.template_manager = SimpleNamespace( jinja_template_content_format="openai" ) serving.tokenizer_manager = SimpleNamespace( tokenizer=FakeTokenizer(), server_args=SimpleNamespace(openai_flatten_tool_role_text_content=True), ) request = ChatCompletionRequest( model="test-model", messages=[ { "role": "tool", "tool_call_id": "call-1", "content": [ {"type": "text", "text": "first line"}, {"type": "text", "text": "second line"}, ], } ], ) serving._apply_jinja_template(request, tools=None, is_multimodal=False) self.assertEqual( captured["messages"][0]["content"], "first line\nsecond line", ) def test_glm_detectors_preserve_argument_value_whitespace(self): _install_function_call_stubs() from sglang.srt.function_call.glm4_moe_detector import Glm4MoeDetector from sglang.srt.function_call.glm47_moe_detector import Glm47MoeDetector for detector_cls in (Glm4MoeDetector, Glm47MoeDetector): with self.subTest(detector=detector_cls.__name__): arguments = detector_cls()._parse_argument_pairs( [(" payload ", " keep surrounding spaces ")], func_name="missing_tool", tools=[], ) self.assertEqual( arguments["payload"], " keep surrounding spaces ", ) class TestParaImageUrlAlignment(unittest.TestCase): def test_image_url_string_is_coerced_to_url_object(self): request = ChatCompletionRequest( model="test-model", messages=[ { "role": "user", "content": [ { "type": "image_url", "image_url": "https://example.test/image.png", } ], } ], ) image_part = request.messages[0].content[0] self.assertEqual(image_part.image_url.url, "https://example.test/image.png") class TestParaPayloadTooLargeAlignment(unittest.TestCase): def test_tokenizer_manager_raises_payload_too_large_for_glm_input_overflow(self): source = ( _REPO_ROOT / "python/sglang/srt/managers/tokenizer_manager.py" ).read_text() # ValueError subclass: raw endpoints that only catch ValueError still # return an error; the OpenAI layer catches it first for the 413 form. self.assertIn("class PayloadTooLargeError(ValueError):", source) self.assertIn('getattr(self.server_args, "openai_glm_compat", False)', source) self.assertIn('"glm" in self.model_path.lower()', source) self.assertIn("raise PayloadTooLargeError(error_msg)", source) def test_serving_base_maps_payload_too_large_to_http_413(self): source = ( _REPO_ROOT / "python/sglang/srt/entrypoints/openai/serving_base.py" ).read_text() self.assertIn( "from sglang.srt.managers.tokenizer_manager import PayloadTooLargeError", source, ) self.assertIn("except PayloadTooLargeError as e:", source) self.assertIn('err_type="PayloadTooLargeError"', source) self.assertIn("status_code=413", source) # PayloadTooLargeError subclasses ValueError, so its except clause must # come first or the ValueError clause swallows it into a 400. self.assertLess( source.index("except PayloadTooLargeError as e:"), source.index("except ValueError as e:"), ) def test_scheduler_over_length_abort_unified_with_payload_too_large(self): """The scheduler-side length rejection must produce the same client format as the TokenizerManager-side PayloadTooLargeError (413).""" from types import SimpleNamespace from sglang.srt.managers.utils import validate_input_length req = SimpleNamespace(origin_input_ids=list(range(100))) error_msg = validate_input_length( req, max_req_input_len=100, allow_auto_truncate=False ) self.assertEqual( error_msg, "The input (100 tokens) is longer than the model's context " "length (100 tokens).", ) scheduler_source = ( _REPO_ROOT / "python/sglang/srt/managers/scheduler.py" ).read_text() self.assertIn( 'err_type="PayloadTooLargeError"', scheduler_source, ) self.assertIn( "status_code=HTTPStatus.REQUEST_ENTITY_TOO_LARGE", scheduler_source, ) class TestParaStreamingErrorAlignment(unittest.TestCase): def test_serving_base_builds_http_error_from_first_streaming_chunk(self): _install_actual_serving_base_stubs() from sglang.srt.entrypoints.openai.serving_base import OpenAIServingBase class ConcreteServingBase(OpenAIServingBase): def _request_id_prefix(self): return "test-" def _convert_to_internal_request(self, request, raw_request=None): return request, request serving = ConcreteServingBase.__new__(ConcreteServingBase) response = serving.create_error_response_from_first_streaming_chunk( 'data: {"error": {"message": "too large", "type": ' '"PayloadTooLargeError", "code": 413, "param": "messages"}}\n\n' ) self.assertEqual(response.status_code, 413) self.assertEqual(response.content["message"], "too large") self.assertEqual(response.content["type"], "PayloadTooLargeError") self.assertEqual(response.content["param"], "messages") def test_chat_streaming_prefetch_returns_http_error_before_sse(self): _install_serving_chat_stubs() from sglang.srt.entrypoints.openai.serving_chat import OpenAIServingChat async def error_generator(): yield ( 'data: {"error": {"message": "too large", "type": ' '"PayloadTooLargeError", "code": 413}}\n\n' ) serving = OpenAIServingChat.__new__(OpenAIServingChat) serving.tokenizer_manager = SimpleNamespace( server_args=SimpleNamespace(openai_streaming_error_preflight=True), create_abort_task=lambda _adapted_request: None ) serving._generate_chat_stream = ( lambda adapted_request, request, raw_request: error_generator() ) response = asyncio.run( serving._handle_streaming_request( adapted_request=SimpleNamespace(), request=ChatCompletionRequest( model="test-model", messages=[{"role": "user", "content": "hello"}], stream=True, ), raw_request=None, ) ) self.assertEqual(response.status_code, 413) self.assertEqual(response.content["message"], "too large") class TestParaUsageAlignment(unittest.TestCase): def test_usage_processor_aggregates_reasoning_tokens(self): usage = UsageProcessor.calculate_response_usage( [ { "meta_info": { "prompt_tokens": 10, "completion_tokens": 3, "reasoning_tokens": 2, } }, { "meta_info": { "prompt_tokens": 10, "completion_tokens": 4, "reasoning_tokens": 5, } }, ], n_choices=2, ) self.assertEqual(usage.prompt_tokens, 10) self.assertEqual(usage.completion_tokens, 7) self.assertEqual(usage.reasoning_tokens, 7) def test_usage_processor_reports_glm_completion_token_details(self): usage = UsageProcessor.calculate_streaming_usage( prompt_tokens={0: 10, 1: 10}, completion_tokens={0: 3, 1: 4}, reasoning_tokens={0: 2, 1: 5}, cached_tokens={}, n_choices=2, use_completion_details=True, ) self.assertEqual(usage.prompt_tokens, 10) self.assertEqual(usage.completion_tokens, 7) self.assertIsNotNone(usage.completion_tokens_details) self.assertEqual(usage.completion_tokens_details.reasoning_tokens, 7) def test_chat_serving_passes_glm_completion_detail_flag(self): source = ( _REPO_ROOT / "python/sglang/srt/entrypoints/openai/serving_chat.py" ).read_text() self.assertIn("reasoning_tokens = {}", source) self.assertIn('reasoning_tokens[index] = content["meta_info"].get(', source) self.assertGreaterEqual(source.count("use_completion_details=self.is_glm"), 3) if __name__ == "__main__": unittest.main(verbosity=2)