chenchenghao/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[PD] Support KV transfer with mooncake (#4880)

Browse Source 
Signed-off-by: Shangming Cai <caishangming@linux.alibaba.com>
Co-authored-by: Shangming Cai <caishangming@linux.alibaba.com>
Co-authored-by: Xuchun Shang <xuchun.shang@linux.alibaba.com>
Co-authored-by: shangmingc <csmthu@gmail.com>
This commit is contained in:
Teng Ma
2025-04-10 14:23:23 +08:00
committed by GitHub
parent f730362ee2
commit 4c31ae9f6d
8 changed files with 571 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
python/sglang/srt/managers/io_struct.py
View File

@@ -95,6 +95,10 @@ class GenerateReqInput:
# Whether to return hidden states
return_hidden_states: bool = False
# For disaggregated inference
bootstrap_host: Optional[str] = None
bootstrap_room: Optional[int] = None
def normalize_batch_and_arguments(self):
"""
Normalize the batch size and arguments for the request.
@@ -435,6 +439,10 @@ class TokenizedGenerateReqInput:
# Whether to return hidden states
return_hidden_states: bool = False
# For disaggregated inference
bootstrap_host: Optional[str] = None
bootstrap_room: Optional[int] = None
@dataclass
class EmbeddingReqInput: