[CUDA Graph] save cuda graph memory by using next_token_logits_buffer (#8579)

2025-08-03 03:06:47 -07:00
parent 7a91330149
commit cb099d2095
5 changed files with 36 additions and 1 deletions
--- a/python/sglang/srt/two_batch_overlap.py
+++ b/python/sglang/srt/two_batch_overlap.py
@@ -564,6 +564,7 @@ class TboForwardBatchPreparer:
                mm_inputs=None,
                top_logprobs_nums=None,
                token_ids_logprobs=None,
+                next_token_logits_buffer=None,
            )
        )