Support spec decoding when LoRA is applied to target model (#12903)

2025-11-16 13:20:23 -08:00
parent 2b8b9d8496
commit 254f62d879
13 changed files with 363 additions and 339 deletions
--- a/python/sglang/srt/model_executor/cuda_graph_runner.py
+++ b/python/sglang/srt/model_executor/cuda_graph_runner.py
@@ -308,7 +308,10 @@ class CudaGraphRunner:
            set_torch_compile_config()

        if self.model_runner.server_args.enable_lora:
-            self.model_runner.lora_manager.init_cuda_graph_batch_info(self.max_bs)
+            self.model_runner.lora_manager.init_cuda_graph_batch_info(
+                max_bs_in_cuda_graph=self.max_bs,
+                num_tokens_per_bs=self.num_tokens_per_bs,
+            )

        # Graph inputs
        with torch.device(self.device):