[feat] support minimum token load balance in dp attention (#7379)

This commit is contained in:
Guanhua Wang
2025-08-03 15:46:47 +08:00
committed by GitHub
parent b0add2da00
commit f7b2853ff8
8 changed files with 271 additions and 6 deletions

View File

@@ -523,6 +523,9 @@ class TokenizedGenerateReqInput:
# For data parallel rank routing
data_parallel_rank: Optional[int] = None
# For dp balance
dp_balance_id: int = -1
@dataclass
class EmbeddingReqInput:
@@ -648,6 +651,8 @@ class TokenizedEmbeddingReqInput:
token_type_ids: List[int]
# Dummy sampling params for compatibility
sampling_params: SamplingParams
# For dp balance
dp_balance_id: int = -1
@dataclass