Teak mem fraction (#20)

2024-01-17 04:43:17 -08:00
parent bf51ddc6e5
commit f9d723816a
2 changed files with 8 additions and 4 deletions
@@ -278,7 +278,7 @@ class ModelRunner:
                load_format=self.load_format,
                revision=None,
            )
-        self.model = model
+        self.model = model.eval()

    def profile_max_num_token(self, total_gpu_memory):
        available_gpu_memory = get_available_gpu_memory(