feat: throttle requests at scheduler based on --max_queued_requests (#7565)

2025-07-28 22:32:33 +08:00
parent b582159246
commit 747dd45077
10 changed files with 218 additions and 6 deletions
@@ -130,6 +130,10 @@ class TpModelWorker:
            self.model_runner.req_to_token_pool.size,
        )
        assert self.max_running_requests > 0, "max_running_request is zero"
+        self.max_queued_requests = server_args.max_queued_requests
+        assert (
+            self.max_running_requests > 0
+        ), "max_queued_requests is zero. We need to be at least 1 to schedule a request."
        self.max_req_len = min(
            self.model_config.context_len - 1,
            self.max_total_num_tokens - 1,
@@ -165,6 +169,7 @@ class TpModelWorker:
            self.max_total_num_tokens,
            self.max_prefill_tokens,
            self.max_running_requests,
+            self.max_queued_requests,
            self.max_req_len,
            self.max_req_input_len,
            self.random_seed,