Fix sending all requests to the first rank in DP attention (#12832)

This commit is contained in:
fzyzcjy
2025-11-09 00:18:53 +08:00
committed by GitHub
parent 243ea585fc
commit 2b6c4257a0

View File

@@ -902,10 +902,14 @@ class TokenizerManager(TokenizerCommunicatorMixin):
Current policy:
- Respect explicit server flag `enable_tokenizer_batch_encode`.
- Or, if no request has text or multimodal input (all use pre-tokenized input_ids or input_embeds), batch the requests without tokenization.
- Batch tokenization does not support DP attention yet, and it will make everything goes to the first rank currently
"""
return batch_size > 0 and (
self.server_args.enable_tokenizer_batch_encode
or not self._batch_has_text(batch_size, requests)
or (
(not self.server_args.enable_dp_attention)
and (not self._batch_has_text(batch_size, requests))
)
)
def _send_one_request(