Support multi-thread model weight loading (#7277)

2025-06-24 10:39:10 -07:00
parent 8ecad0b16f
commit 9f1787fa60
4 changed files with 143 additions and 10 deletions
@@ -547,6 +547,7 @@ class ModelRunner:
        self.load_config = LoadConfig(
            load_format=self.server_args.load_format,
            download_dir=self.server_args.download_dir,
+            model_loader_extra_config=self.server_args.model_loader_extra_config,
        )
        if self.server_args.load_format == "gguf":
            monkey_patch_vllm_gguf_config()