feat: add request queued timeout (#17143)

Co-authored-by: qiuxuan.lzw <qiuxuan.lzw@alibaba-inc.com>
Co-authored-by: Xiaoyu Zhang <35585791+BBuf@users.noreply.github.com>

python/sglang/srt/environ.py

@@ -242,6 +242,7 @@ class Envs:
     SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES = EnvInt(30)
     SGLANG_PREFILL_DELAYER_TOKEN_USAGE_LOW_WATERMARK = EnvFloat(None)
     SGLANG_DATA_PARALLEL_BUDGET_INTERVAL = EnvInt(1)
+    SGLANG_QUEUED_TIMEOUT_MS = EnvInt(-1)
 
     # Test: pd-disaggregation
     SGLANG_TEST_PD_DISAGG_BACKEND = EnvStr("mooncake")

python/sglang/srt/managers/scheduler.py

@@ -1671,6 +1671,33 @@ class Scheduler(
         )
         return req_to_abort.rid == recv_req.rid
 
+    def _abort_on_queued_timeout(self):
+        if (timeout_ms := envs.SGLANG_QUEUED_TIMEOUT_MS.get()) <= 0:
+            return
+
+        deleted_reqs = set()
+        deadline = time.perf_counter() - (timeout_ms / 1000.0)
+        for req in self.waiting_queue:
+            entry_time = req.time_stats.wait_queue_entry_time
+            if 0 < entry_time < deadline:
+                self.send_to_tokenizer.send_output(
+                    AbortReq(
+                        finished_reason={
+                            "type": "abort",
+                            "status_code": HTTPStatus.SERVICE_UNAVAILABLE,
+                            "message": "Request queue timeout reached.",
+                        },
+                        rid=req.rid,
+                    ),
+                    req,
+                )
+                deleted_reqs.add(req)
+
+        if deleted_reqs:
+            self.waiting_queue = [
+                req for req in self.waiting_queue if req not in deleted_reqs
+            ]
+
     def handle_embedding_request(
         self,
         recv_req: TokenizedEmbeddingReqInput,
@@ -1739,6 +1766,7 @@ class Scheduler(
             self.handle_embedding_request(tokenized_req)
 
     def get_next_batch_to_run(self) -> Optional[ScheduleBatch]:
+        self._abort_on_queued_timeout()
         if self.dllm_config is not None:
             if self.chunked_req is not None and self.chunked_req.finished():
                 self.chunked_req = None

python/sglang/srt/utils/request_logger.py

@@ -130,7 +130,7 @@ class RequestLogger:
         if not self.log_requests:
             return
 
-        e2e_latency_ms = out["meta_info"]["e2e_latency"] * 1000
+        e2e_latency_ms = out["meta_info"].get("e2e_latency", 0) * 1000
         if self.log_exceeded_ms > 0 and e2e_latency_ms < self.log_exceeded_ms:
             return