chenchenghao/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[PD] Support KV transfer with mooncake (#4880)

Browse Source 
Signed-off-by: Shangming Cai <caishangming@linux.alibaba.com>
Co-authored-by: Shangming Cai <caishangming@linux.alibaba.com>
Co-authored-by: Xuchun Shang <xuchun.shang@linux.alibaba.com>
Co-authored-by: shangmingc <csmthu@gmail.com>
This commit is contained in:
Teng Ma
2025-04-10 14:23:23 +08:00
committed by GitHub
parent f730362ee2
commit 4c31ae9f6d
8 changed files with 571 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
python/sglang/srt/managers/schedule_batch.py
View File

@@ -390,6 +390,8 @@ class Req:
custom_logit_processor: Optional[str] = None,
return_hidden_states: bool = False,
eos_token_ids: Optional[Set[int]] = None,
bootstrap_host: Optional[str] = None,
bootstrap_room: Optional[int] = None,
):
# Input and output info
self.rid = rid
@@ -523,8 +525,8 @@ class Req:
self.lora_path = lora_path
# For disaggregation
self.bootstrap_host: str = "0.0.0.0"
self.bootstrap_room: Optional[int] = None
self.bootstrap_host: str = bootstrap_host
self.bootstrap_room: Optional[int] = bootstrap_room
self.disagg_kv_sender: Optional[KVSender] = None
# used for warmup because we don't have a pair yet when init