[4/N]DP refactor: support watching mode get_load and shortest queue strategy (#10201)

This commit is contained in:
Liangsheng Yin
2025-09-15 10:06:08 +08:00
committed by GitHub
parent ca63f075b7
commit 305c9e8c2d
12 changed files with 202 additions and 44 deletions
+18
View File
@@ -1374,3 +1374,21 @@ class BlockReqType(Enum):
@dataclass
class BlockReqInput:
type: BlockReqType
@dataclass
class GetLoadReqInput:
pass
@dataclass
class GetLoadReqOutput:
dp_rank: int
num_reqs: int
num_waiting_reqs: int
num_tokens: int
@dataclass
class WatchLoadUpdateReq:
loads: List[GetLoadReqOutput]