Support start up LoRA server without initial adapters (#8019)
This commit is contained in:
@@ -264,7 +264,7 @@ class CudaGraphRunner:
|
||||
if self.enable_torch_compile:
|
||||
set_torch_compile_config()
|
||||
|
||||
if self.model_runner.server_args.lora_paths is not None:
|
||||
if self.model_runner.server_args.enable_lora:
|
||||
self.model_runner.lora_manager.init_cuda_graph_batch_info(self.max_bs)
|
||||
|
||||
# Graph inputs
|
||||
@@ -510,11 +510,10 @@ class CudaGraphRunner:
|
||||
spec_info.capture_hidden_mode if spec_info else CaptureHiddenMode.NULL
|
||||
)
|
||||
|
||||
if self.model_runner.server_args.lora_paths is not None:
|
||||
# Currently, if the lora_path in `lora_paths` is None, the lora backend will use a
|
||||
# different logic to handle lora, so we need to set `lora_paths` to a list of non-None
|
||||
# values if lora is enabled.
|
||||
lora_paths = [next(iter(self.model_runner.server_args.lora_paths))] * bs
|
||||
if self.model_runner.server_args.enable_lora:
|
||||
# It is safe to capture CUDA graph using empty LoRA path, as the LoRA kernels will always be launched whenever
|
||||
# `--enable-lora` is set to True (and return immediately if the LoRA path is empty for perf optimization).
|
||||
lora_paths = [None] * bs
|
||||
else:
|
||||
lora_paths = None
|
||||
|
||||
|
||||
Reference in New Issue
Block a user