feat: throttle requests at scheduler based on --max_queued_requests (#7565)

2025-07-28 07:32:33 -07:00
parent b582159246
commit 747dd45077
10 changed files with 218 additions and 6 deletions
--- a/python/sglang/srt/managers/io_struct.py
+++ b/python/sglang/srt/managers/io_struct.py
@@ -911,6 +911,8 @@ class AbortReq:
    rid: str = ""
    # Whether to abort all requests
    abort_all: bool = False
+    # The finished reason data
+    finished_reason: Optional[Dict[str, Any]] = None


@dataclass