Clean up readme and arguments of chunked prefill (#1022)

2024-08-11 01:18:52 -07:00
parent 33d61356b8
commit a97df79124
3 changed files with 30 additions and 34 deletions
@@ -118,11 +118,7 @@ class ModelTpServer:
                    trust_remote_code=server_args.trust_remote_code,
                )
        self.max_total_num_tokens = self.model_runner.max_total_num_tokens
-        self.max_prefill_tokens = (
-            16384
-            if server_args.max_prefill_tokens is None
-            else server_args.max_prefill_tokens
-        )
+        self.max_prefill_tokens = server_args.max_prefill_tokens
        self.max_running_requests = min(
            (
                self.max_total_num_tokens // 2