Support spec decoding when LoRA is applied to target model (#12903)

This commit is contained in:
Lifu Huang
2025-11-16 13:20:23 -08:00
committed by GitHub
parent 2b8b9d8496
commit 254f62d879
13 changed files with 363 additions and 339 deletions
+7
View File
@@ -3874,6 +3874,13 @@ class ServerArgs:
)
if self.enable_lora:
# Validate compatibility with speculative decoding
if self.speculative_algorithm not in ["NGRAM", None]:
raise ValueError(
"Currently LoRA is only compatible with NGRAM speculative decoding."
)
# Parse lora_paths
if isinstance(self.lora_paths, list):
lora_paths = self.lora_paths
self.lora_paths = []