Unify over-length errors into the PayloadTooLargeError 413 format
Over-long inputs produced two different client errors depending on
which bound rejected them: the TokenizerManager pre-check (raw
context_len) returned 413 PayloadTooLargeError ('The input (N tokens)
is longer than the model's context length (M tokens).'), while inputs
between that and the scheduler's stricter effective limit hit
validate_input_length and returned 400 BAD_REQUEST with different
wording (and a confusing 'X exceeds X' message since the check is >=).
Unify on the 413 format end to end:
- validate_input_length wording now matches the TokenizerManager
message, reporting the effective per-request limit.
- set_finish_with_abort takes status_code/err_type; the scheduler
length-rejection sites abort with REQUEST_ENTITY_TOO_LARGE +
PayloadTooLargeError. The batch handler previously queued the
over-long request WITHOUT marking it aborted (it proceeded to
prefill) — also fixed.
- Non-streaming aborts with 413 raise PayloadTooLargeError (now a
ValueError subclass so raw /generate-style endpoints that only
catch ValueError still respond; the OpenAI layer's except clause
is reordered to win and emit the 413 format).
- Streaming abort responses prefer the scheduler-provided err_type
over the HTTPStatus name.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
@@ -602,7 +602,9 @@ class TestParaPayloadTooLargeAlignment(unittest.TestCase):
|
||||
_REPO_ROOT / "python/sglang/srt/managers/tokenizer_manager.py"
|
||||
).read_text()
|
||||
|
||||
self.assertIn("class PayloadTooLargeError(Exception):", source)
|
||||
# ValueError subclass: raw endpoints that only catch ValueError still
|
||||
# return an error; the OpenAI layer catches it first for the 413 form.
|
||||
self.assertIn("class PayloadTooLargeError(ValueError):", source)
|
||||
self.assertIn('getattr(self.server_args, "openai_glm_compat", False)', source)
|
||||
self.assertIn('"glm" in self.model_path.lower()', source)
|
||||
self.assertIn("raise PayloadTooLargeError(error_msg)", source)
|
||||
@@ -619,6 +621,41 @@ class TestParaPayloadTooLargeAlignment(unittest.TestCase):
|
||||
self.assertIn("except PayloadTooLargeError as e:", source)
|
||||
self.assertIn('err_type="PayloadTooLargeError"', source)
|
||||
self.assertIn("status_code=413", source)
|
||||
# PayloadTooLargeError subclasses ValueError, so its except clause must
|
||||
# come first or the ValueError clause swallows it into a 400.
|
||||
self.assertLess(
|
||||
source.index("except PayloadTooLargeError as e:"),
|
||||
source.index("except ValueError as e:"),
|
||||
)
|
||||
|
||||
def test_scheduler_over_length_abort_unified_with_payload_too_large(self):
|
||||
"""The scheduler-side length rejection must produce the same client
|
||||
format as the TokenizerManager-side PayloadTooLargeError (413)."""
|
||||
from types import SimpleNamespace
|
||||
|
||||
from sglang.srt.managers.utils import validate_input_length
|
||||
|
||||
req = SimpleNamespace(origin_input_ids=list(range(100)))
|
||||
error_msg = validate_input_length(
|
||||
req, max_req_input_len=100, allow_auto_truncate=False
|
||||
)
|
||||
self.assertEqual(
|
||||
error_msg,
|
||||
"The input (100 tokens) is longer than the model's context "
|
||||
"length (100 tokens).",
|
||||
)
|
||||
|
||||
scheduler_source = (
|
||||
_REPO_ROOT / "python/sglang/srt/managers/scheduler.py"
|
||||
).read_text()
|
||||
self.assertIn(
|
||||
'err_type="PayloadTooLargeError"',
|
||||
scheduler_source,
|
||||
)
|
||||
self.assertIn(
|
||||
"status_code=HTTPStatus.REQUEST_ENTITY_TOO_LARGE",
|
||||
scheduler_source,
|
||||
)
|
||||
|
||||
|
||||
class TestParaStreamingErrorAlignment(unittest.TestCase):
|
||||
|
||||
Reference in New Issue
Block a user