chenchenghao/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 11 Packages Projects Releases Wiki Activity

Rename lora_path to lora_id in batches (#8437)

Browse Source
This commit is contained in:
Baizhou Zhang
2025-08-03 21:08:28 -07:00
committed by GitHub
parent 3b87a9e8ae
commit f2d68ded6d
8 changed files with 27 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
python/sglang/srt/model_executor/cuda_graph_runner.py
View File

@@ -576,11 +576,11 @@ class CudaGraphRunner:
)
if self.model_runner.server_args.enable_lora:
# It is safe to capture CUDA graph using empty LoRA path, as the LoRA kernels will always be launched whenever
# `--enable-lora` is set to True (and return immediately if the LoRA path is empty for perf optimization).
lora_paths = [None] * bs
# It is safe to capture CUDA graph using empty LoRA id, as the LoRA kernels will always be launched whenever
# `--enable-lora` is set to True (and return immediately if the LoRA id is empty for perf optimization).
lora_ids = [None] * bs
else:
lora_paths = None
lora_ids = None
forward_batch = ForwardBatch(
forward_mode=self.capture_forward_mode,
@@ -607,11 +607,11 @@ class CudaGraphRunner:
capture_hidden_mode=self.capture_hidden_mode,
num_token_non_padded=self.num_token_non_padded,
global_forward_mode=self.capture_forward_mode,
lora_paths=lora_paths,
lora_ids=lora_ids,
)
self.tbo_plugin.capture_one_batch_size(forward_batch, num_tokens=num_tokens)
if lora_paths is not None:
if lora_ids is not None:
self.model_runner.lora_manager.prepare_lora_batch(forward_batch)
# Attention backend