From 75d7d8772ef8e93410db879607ae83d5fffc3101 Mon Sep 17 00:00:00 2001 From: leavelet Date: Thu, 11 Jun 2026 08:01:02 +0000 Subject: [PATCH] Unify over-length errors into the PayloadTooLargeError 413 format MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Over-long inputs produced two different client errors depending on which bound rejected them: the TokenizerManager pre-check (raw context_len) returned 413 PayloadTooLargeError ('The input (N tokens) is longer than the model's context length (M tokens).'), while inputs between that and the scheduler's stricter effective limit hit validate_input_length and returned 400 BAD_REQUEST with different wording (and a confusing 'X exceeds X' message since the check is >=). Unify on the 413 format end to end: - validate_input_length wording now matches the TokenizerManager message, reporting the effective per-request limit. - set_finish_with_abort takes status_code/err_type; the scheduler length-rejection sites abort with REQUEST_ENTITY_TOO_LARGE + PayloadTooLargeError. The batch handler previously queued the over-long request WITHOUT marking it aborted (it proceeded to prefill) — also fixed. - Non-streaming aborts with 413 raise PayloadTooLargeError (now a ValueError subclass so raw /generate-style endpoints that only catch ValueError still respond; the OpenAI layer's except clause is reordered to win and emit the 413 format). - Streaming abort responses prefer the scheduler-provided err_type over the HTTPStatus name. Co-Authored-By: Claude Fable 5 --- .../srt/entrypoints/openai/serving_base.py | 13 ++++--- .../srt/entrypoints/openai/serving_chat.py | 3 ++ .../entrypoints/openai/serving_completions.py | 3 ++ python/sglang/srt/managers/schedule_batch.py | 11 ++++-- python/sglang/srt/managers/scheduler.py | 13 ++++++- .../sglang/srt/managers/tokenizer_manager.py | 25 +++++++++--- python/sglang/srt/managers/utils.py | 9 +++-- .../openai/test_para_serving_protocol.py | 39 ++++++++++++++++++- 8 files changed, 95 insertions(+), 21 deletions(-) diff --git a/python/sglang/srt/entrypoints/openai/serving_base.py b/python/sglang/srt/entrypoints/openai/serving_base.py index c219a98c5..749c19dc0 100644 --- a/python/sglang/srt/entrypoints/openai/serving_base.py +++ b/python/sglang/srt/entrypoints/openai/serving_base.py @@ -130,18 +130,19 @@ class OpenAIServingBase(ABC): return self.create_error_response( message=e.detail, err_type=str(e.status_code), status_code=e.status_code ) + except PayloadTooLargeError as e: + # Must precede ValueError: PayloadTooLargeError subclasses it. + return self.create_error_response( + message=str(e), + err_type="PayloadTooLargeError", + status_code=413, + ) except ValueError as e: return self.create_error_response( message=str(e), err_type="BadRequest", status_code=400, ) - except PayloadTooLargeError as e: - return self.create_error_response( - message=str(e), - err_type="PayloadTooLargeError", - status_code=413, - ) except DS32EncodingError as e: logger.info(f"DS32EncodingError: {e}") return self.create_error_response( diff --git a/python/sglang/srt/entrypoints/openai/serving_chat.py b/python/sglang/srt/entrypoints/openai/serving_chat.py index 008761f1e..78a602136 100644 --- a/python/sglang/srt/entrypoints/openai/serving_chat.py +++ b/python/sglang/srt/entrypoints/openai/serving_chat.py @@ -970,6 +970,9 @@ class OpenAIServingChat(OpenAIServingBase): err_type, status_code = ( self._streaming_http_error_type_and_status(code) ) + # Prefer the scheduler-provided error type (e.g. + # PayloadTooLargeError) over the HTTPStatus name. + err_type = finish_reason.get("err_type") or err_type error = self.create_streaming_error_response( finish_reason.get("message", "Generation aborted."), err_type, diff --git a/python/sglang/srt/entrypoints/openai/serving_completions.py b/python/sglang/srt/entrypoints/openai/serving_completions.py index 874047c52..205713475 100644 --- a/python/sglang/srt/entrypoints/openai/serving_completions.py +++ b/python/sglang/srt/entrypoints/openai/serving_completions.py @@ -281,6 +281,9 @@ class OpenAIServingCompletion(OpenAIServingBase): err_type, status_code = self._streaming_http_error_type_and_status( code ) + # Prefer the scheduler-provided error type (e.g. + # PayloadTooLargeError) over the HTTPStatus name. + err_type = finish_reason.get("err_type") or err_type error = self.create_streaming_error_response( finish_reason.get("message", "Generation aborted."), err_type, diff --git a/python/sglang/srt/managers/schedule_batch.py b/python/sglang/srt/managers/schedule_batch.py index ca7920732..6b6cbf0f6 100644 --- a/python/sglang/srt/managers/schedule_batch.py +++ b/python/sglang/srt/managers/schedule_batch.py @@ -1203,7 +1203,12 @@ class Req(ReqDllmMixin): self.extend_input_len, ) - def set_finish_with_abort(self, error_msg: str): + def set_finish_with_abort( + self, + error_msg: str, + status_code: HTTPStatus = HTTPStatus.BAD_REQUEST, + err_type: str = "BadRequestError", + ): if get_tensor_model_parallel_rank() == 0: logger.error(f"{error_msg}, {self.rid=}") self.multimodal_inputs = None @@ -1211,9 +1216,7 @@ class Req(ReqDllmMixin): self.origin_input_ids = [0] # set it to one token to skip the long prefill self.return_logprob = False self.logprob_start_len = -1 - self.to_finish = FINISH_ABORT( - error_msg, HTTPStatus.BAD_REQUEST, "BadRequestError" - ) + self.to_finish = FINISH_ABORT(error_msg, status_code, err_type) def __repr__(self): return ( diff --git a/python/sglang/srt/managers/scheduler.py b/python/sglang/srt/managers/scheduler.py index 1cf95e96c..eba94e203 100644 --- a/python/sglang/srt/managers/scheduler.py +++ b/python/sglang/srt/managers/scheduler.py @@ -1976,7 +1976,11 @@ class Scheduler( self.server_args.allow_auto_truncate, ) if error_msg: - req.set_finish_with_abort(error_msg) + req.set_finish_with_abort( + error_msg, + status_code=HTTPStatus.REQUEST_ENTITY_TOO_LARGE, + err_type="PayloadTooLargeError", + ) self._add_request_to_queue(req) return @@ -2225,6 +2229,13 @@ class Scheduler( self.server_args.allow_auto_truncate, ) if error_msg: + # NOTE: this path previously queued the over-long request without + # marking it aborted, letting it proceed to prefill. + req.set_finish_with_abort( + error_msg, + status_code=HTTPStatus.REQUEST_ENTITY_TOO_LARGE, + err_type="PayloadTooLargeError", + ) self._add_request_to_queue(req) return diff --git a/python/sglang/srt/managers/tokenizer_manager.py b/python/sglang/srt/managers/tokenizer_manager.py index 7062d8e68..a7b02da60 100644 --- a/python/sglang/srt/managers/tokenizer_manager.py +++ b/python/sglang/srt/managers/tokenizer_manager.py @@ -122,8 +122,13 @@ asyncio.set_event_loop_policy(uvloop.EventLoopPolicy()) _REQUEST_STATE_WAIT_TIMEOUT = envs.SGLANG_REQUEST_STATE_WAIT_TIMEOUT.get() -class PayloadTooLargeError(Exception): - """Exception raised when a request payload exceeds the model context length.""" +class PayloadTooLargeError(ValueError): + """Exception raised when a request payload exceeds the model context length. + + Subclasses ValueError so callers that only handle ValueError (e.g. the raw + /generate endpoint) still return an error response; the OpenAI serving + layer catches it first to produce the 413 PayloadTooLargeError format. + """ logger = logging.getLogger(__name__) @@ -1207,12 +1212,20 @@ class TokenizerManager(TokenizerCommunicatorMixin, TokenizerManagerMultiItemMixi # Check if this was an abort/error created by scheduler if isinstance(out["meta_info"].get("finish_reason"), dict): finish_reason = out["meta_info"]["finish_reason"] - if ( - finish_reason.get("type") == "abort" - and finish_reason.get("status_code") - == HTTPStatus.BAD_REQUEST + if finish_reason.get("type") == "abort" and finish_reason.get( + "status_code" + ) in ( + HTTPStatus.BAD_REQUEST, + HTTPStatus.REQUEST_ENTITY_TOO_LARGE, ): if not is_stream: + if ( + finish_reason.get("status_code") + == HTTPStatus.REQUEST_ENTITY_TOO_LARGE + ): + raise PayloadTooLargeError( + finish_reason["message"] + ) raise ValueError(finish_reason["message"]) else: yield out diff --git a/python/sglang/srt/managers/utils.py b/python/sglang/srt/managers/utils.py index ba7773300..67ee9786c 100644 --- a/python/sglang/srt/managers/utils.py +++ b/python/sglang/srt/managers/utils.py @@ -133,10 +133,13 @@ def validate_input_length( req.origin_input_ids = req.origin_input_ids[:max_req_input_len] return None else: + # Keep the wording identical to the TokenizerManager-side + # PayloadTooLargeError so clients see one over-length format + # regardless of which bound (raw context length there, effective + # per-request limit here) rejected the request. error_msg = ( - f"Input length ({len(req.origin_input_ids)} tokens) exceeds " - f"the maximum allowed length ({max_req_input_len} tokens). " - f"Use a shorter input or enable --allow-auto-truncate." + f"The input ({len(req.origin_input_ids)} tokens) is longer " + f"than the model's context length ({max_req_input_len} tokens)." ) return error_msg diff --git a/test/registered/unit/entrypoints/openai/test_para_serving_protocol.py b/test/registered/unit/entrypoints/openai/test_para_serving_protocol.py index 1e9ecc1f4..94ed39e40 100644 --- a/test/registered/unit/entrypoints/openai/test_para_serving_protocol.py +++ b/test/registered/unit/entrypoints/openai/test_para_serving_protocol.py @@ -602,7 +602,9 @@ class TestParaPayloadTooLargeAlignment(unittest.TestCase): _REPO_ROOT / "python/sglang/srt/managers/tokenizer_manager.py" ).read_text() - self.assertIn("class PayloadTooLargeError(Exception):", source) + # ValueError subclass: raw endpoints that only catch ValueError still + # return an error; the OpenAI layer catches it first for the 413 form. + self.assertIn("class PayloadTooLargeError(ValueError):", source) self.assertIn('getattr(self.server_args, "openai_glm_compat", False)', source) self.assertIn('"glm" in self.model_path.lower()', source) self.assertIn("raise PayloadTooLargeError(error_msg)", source) @@ -619,6 +621,41 @@ class TestParaPayloadTooLargeAlignment(unittest.TestCase): self.assertIn("except PayloadTooLargeError as e:", source) self.assertIn('err_type="PayloadTooLargeError"', source) self.assertIn("status_code=413", source) + # PayloadTooLargeError subclasses ValueError, so its except clause must + # come first or the ValueError clause swallows it into a 400. + self.assertLess( + source.index("except PayloadTooLargeError as e:"), + source.index("except ValueError as e:"), + ) + + def test_scheduler_over_length_abort_unified_with_payload_too_large(self): + """The scheduler-side length rejection must produce the same client + format as the TokenizerManager-side PayloadTooLargeError (413).""" + from types import SimpleNamespace + + from sglang.srt.managers.utils import validate_input_length + + req = SimpleNamespace(origin_input_ids=list(range(100))) + error_msg = validate_input_length( + req, max_req_input_len=100, allow_auto_truncate=False + ) + self.assertEqual( + error_msg, + "The input (100 tokens) is longer than the model's context " + "length (100 tokens).", + ) + + scheduler_source = ( + _REPO_ROOT / "python/sglang/srt/managers/scheduler.py" + ).read_text() + self.assertIn( + 'err_type="PayloadTooLargeError"', + scheduler_source, + ) + self.assertIn( + "status_code=HTTPStatus.REQUEST_ENTITY_TOO_LARGE", + scheduler_source, + ) class TestParaStreamingErrorAlignment(unittest.TestCase):