Fix sending all requests to the first rank in DP attention (#12832)
This commit is contained in:
@@ -902,10 +902,14 @@ class TokenizerManager(TokenizerCommunicatorMixin):
|
||||
Current policy:
|
||||
- Respect explicit server flag `enable_tokenizer_batch_encode`.
|
||||
- Or, if no request has text or multimodal input (all use pre-tokenized input_ids or input_embeds), batch the requests without tokenization.
|
||||
- Batch tokenization does not support DP attention yet, and it will make everything goes to the first rank currently
|
||||
"""
|
||||
return batch_size > 0 and (
|
||||
self.server_args.enable_tokenizer_batch_encode
|
||||
or not self._batch_has_text(batch_size, requests)
|
||||
or (
|
||||
(not self.server_args.enable_dp_attention)
|
||||
and (not self._batch_has_text(batch_size, requests))
|
||||
)
|
||||
)
|
||||
|
||||
def _send_one_request(
|
||||
|
||||
Reference in New Issue
Block a user