Support spec decoding when LoRA is applied to target model (#12903)
This commit is contained in:
@@ -3874,6 +3874,13 @@ class ServerArgs:
|
||||
)
|
||||
|
||||
if self.enable_lora:
|
||||
# Validate compatibility with speculative decoding
|
||||
if self.speculative_algorithm not in ["NGRAM", None]:
|
||||
raise ValueError(
|
||||
"Currently LoRA is only compatible with NGRAM speculative decoding."
|
||||
)
|
||||
|
||||
# Parse lora_paths
|
||||
if isinstance(self.lora_paths, list):
|
||||
lora_paths = self.lora_paths
|
||||
self.lora_paths = []
|
||||
|
||||
Reference in New Issue
Block a user