Support multi-node DP attention (#2925)

Co-authored-by: dhou-xai <dhou@x.ai>
2025-01-16 11:15:00 -08:00
parent 58f3f2b840
commit 8b6ce52e92
16 changed files with 287 additions and 137 deletions
--- a/python/sglang/srt/layers/logits_processor.py
+++ b/python/sglang/srt/layers/logits_processor.py
@@ -133,7 +133,7 @@ class LogitsProcessor(nn.Module):

        # Get the last hidden states and last logits for the next token prediction
        if (
-            logits_metadata.forward_mode.is_decode()
+            logits_metadata.forward_mode.is_decode_or_idle()
            or logits_metadata.forward_mode.is_target_verify()
        ):
            last_index = None