Support multi-thread model weight loading (#7277)

2025-06-25 01:39:10 +08:00
parent 8ecad0b16f
commit 9f1787fa60
4 changed files with 143 additions and 10 deletions
--- a/python/sglang/srt/model_executor/model_runner.py
+++ b/python/sglang/srt/model_executor/model_runner.py
@@ -547,6 +547,7 @@ class ModelRunner:
        self.load_config = LoadConfig(
            load_format=self.server_args.load_format,
            download_dir=self.server_args.download_dir,
+            model_loader_extra_config=self.server_args.model_loader_extra_config,
        )
        if self.server_args.load_format == "gguf":
            monkey_patch_vllm_gguf_config()