feat: add priority based scheduling with priority based request acceptance and preemption (#8746)

2025-09-16 17:10:10 -07:00
parent f949ad5794
commit 14fdd52740
16 changed files with 822 additions and 71 deletions
@@ -149,8 +149,8 @@ class TpModelWorker:
        assert self.max_running_requests > 0, "max_running_request is zero"
        self.max_queued_requests = server_args.max_queued_requests
        assert (
-            self.max_queued_requests > 0
-        ), "max_queued_requests is zero. We need to be at least 1 to schedule a request."
+            self.max_queued_requests is None or self.max_queued_requests >= 1
+        ), "If configured, max_queued_requests must be at least 1 for any work to be scheduled."
        self.max_req_len = min(
            self.model_config.context_len - 1,
            self.max_total_num_tokens - 1,