refactor model loader: initial refactor (#664)

2024-07-20 02:18:22 -07:00
parent 39c57317e1
commit 06487f126e
6 changed files with 100 additions and 15 deletions
--- a/python/sglang/srt/model_loader/model_loader.py
+++ b/python/sglang/srt/model_loader/model_loader.py
@@ -91,6 +91,7 @@ def _initialize_model(
        config=model_config.hf_config,
        cache_config=cache_config,
        quant_config=quant_config,
+        efficient_weight_load=True,
        **_get_model_initialization_kwargs(model_class, lora_config, multimodal_config),
    )