feat: add request queued timeout (#17143)

Co-authored-by: qiuxuan.lzw <qiuxuan.lzw@alibaba-inc.com>
Co-authored-by: Xiaoyu Zhang <35585791+BBuf@users.noreply.github.com>

test/registered/scheduler/test_abort.py

@@ -191,5 +191,52 @@ class TestAbortAllWithRetraction(CustomTestCase):
             print("Finished test_abort_all_with_retraction")
 
 
+class TestAbortWithQueueTimeout(CustomTestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.model = DEFAULT_MODEL_NAME_FOR_TEST
+        cls.base_url = DEFAULT_URL_FOR_TEST
+        with envs.SGLANG_QUEUED_TIMEOUT_MS.override(1):
+            cls.process = popen_launch_server(
+                cls.model,
+                cls.base_url,
+                timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
+                other_args=[
+                    "--max-running-requests=1",
+                ],
+            )
+
+    @classmethod
+    def tearDownClass(cls):
+        kill_process_tree(cls.process.pid)
+
+    def _run_decode(self):
+        response = requests.post(
+            self.base_url + "/generate",
+            json={
+                "text": "Today is ",
+                "sampling_params": {
+                    "temperature": 0,
+                    "max_new_tokens": 512,
+                    "ignore_eos": True,
+                },
+            },
+        )
+        return response.json()
+
+    def test_queue_timeout(self):
+        num_requests = 2
+        with ThreadPoolExecutor(num_requests) as executor:
+            futures = [executor.submit(self._run_decode) for _ in range(num_requests)]
+
+            error_count = 0
+            for future in as_completed(futures):
+                result = future.result()
+                if result.get("object") == "error":
+                    error_count += 1
+                    self.assertEqual(result["code"], 503)
+            self.assertEqual(error_count, 1)
+
+
 if __name__ == "__main__":
     unittest.main()