chenchenghao/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 11 Packages Projects Releases Wiki Activity

[PD]: Support Muti Prefill in one node (#5704)

Browse Source 
Co-authored-by: shuaills <shishuaiuoe@gmail.com>
This commit is contained in:
IAN
2025-04-26 00:30:47 +08:00
committed by GitHub
parent 50eda8398e
commit 11e27d0926
6 changed files with 55 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
python/sglang/srt/managers/io_struct.py
View File

@@ -97,6 +97,7 @@ class GenerateReqInput:
# For disaggregated inference
bootstrap_host: Optional[Union[List[str], str]] = None
bootstrap_port: Optional[Union[List[int], int]] = None
bootstrap_room: Optional[Union[List[int], int]] = None
def normalize_batch_and_arguments(self):
@@ -400,6 +401,9 @@ class GenerateReqInput:
bootstrap_host=(
self.bootstrap_host[i] if self.bootstrap_host is not None else None
),
bootstrap_port=(
self.bootstrap_port[i] if self.bootstrap_port is not None else None
),
bootstrap_room=(
self.bootstrap_room[i] if self.bootstrap_room is not None else None
),
@@ -447,6 +451,7 @@ class TokenizedGenerateReqInput:
# For disaggregated inference
bootstrap_host: Optional[str] = None
bootstrap_port: Optional[int] = None
bootstrap_room: Optional[int] = None