[PD]: Support Muti Prefill in one node (#5704)

Co-authored-by: shuaills <shishuaiuoe@gmail.com>
This commit is contained in:
IAN
2025-04-26 00:30:47 +08:00
committed by GitHub
parent 50eda8398e
commit 11e27d0926
6 changed files with 55 additions and 9 deletions

View File

@@ -391,6 +391,7 @@ class Req:
return_hidden_states: bool = False,
eos_token_ids: Optional[Set[int]] = None,
bootstrap_host: Optional[str] = None,
bootstrap_port: Optional[int] = None,
bootstrap_room: Optional[int] = None,
):
# Input and output info
@@ -526,6 +527,7 @@ class Req:
# For disaggregation
self.bootstrap_host: str = bootstrap_host
self.bootstrap_port: Optional[int] = bootstrap_port
self.bootstrap_room: Optional[int] = bootstrap_room
self.disagg_kv_sender: Optional[BaseKVSender] = None