[feat] support minimum token load balance in dp attention (#7379)
This commit is contained in:
@@ -523,6 +523,9 @@ class TokenizedGenerateReqInput:
|
||||
# For data parallel rank routing
|
||||
data_parallel_rank: Optional[int] = None
|
||||
|
||||
# For dp balance
|
||||
dp_balance_id: int = -1
|
||||
|
||||
|
||||
@dataclass
|
||||
class EmbeddingReqInput:
|
||||
@@ -648,6 +651,8 @@ class TokenizedEmbeddingReqInput:
|
||||
token_type_ids: List[int]
|
||||
# Dummy sampling params for compatibility
|
||||
sampling_params: SamplingParams
|
||||
# For dp balance
|
||||
dp_balance_id: int = -1
|
||||
|
||||
|
||||
@dataclass
|
||||
|
||||
Reference in New Issue
Block a user