[Fix] memory leak by overlap + retract (#11981)

Co-authored-by: Liangsheng Yin <lsyincs@gmail.com>
2025-10-23 22:59:23 +08:00
parent 6c18addb6f
commit b0b4f71679
9 changed files with 132 additions and 25 deletions
@@ -569,7 +569,8 @@ class PrefillAdder:
            return self.add_one_req_ignore_eos(req, has_chunked_req)

        total_tokens = req.extend_input_len + min(
-            req.sampling_params.max_new_tokens, CLIP_MAX_NEW_TOKENS
+            max(req.sampling_params.max_new_tokens - len(req.output_ids), 0),
+            CLIP_MAX_NEW_TOKENS,
        )

        # adjusting the input_tokens based on host_hit_length and page_size