[Fix] memory leak by overlap + retract (#11981)
Co-authored-by: Liangsheng Yin <lsyincs@gmail.com>
This commit is contained in:
@@ -885,7 +885,6 @@ class Req:
|
||||
self.temp_input_top_logprobs_idx = None
|
||||
self.extend_logprob_start_len = 0
|
||||
self.is_chunked = 0
|
||||
self.req_pool_idx = None
|
||||
self.mamba_pool_idx = None
|
||||
self.already_computed = 0
|
||||
|
||||
@@ -1482,7 +1481,9 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
|
||||
new_estimate_ratio = (
|
||||
total_decoded_tokens
|
||||
+ envs.SGLANG_RETRACT_DECODE_STEPS.get() * len(self.reqs)
|
||||
) / total_max_new_tokens
|
||||
) / (
|
||||
total_max_new_tokens + 1
|
||||
) # avoid zero division
|
||||
new_estimate_ratio = min(1.0, new_estimate_ratio)
|
||||
|
||||
return retracted_reqs, new_estimate_ratio, []
|
||||
@@ -1780,6 +1781,8 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
|
||||
# Only contain fields that will be used by process_batch_result
|
||||
return ScheduleBatch(
|
||||
reqs=self.reqs,
|
||||
req_to_token_pool=self.req_to_token_pool,
|
||||
req_pool_indices=self.req_pool_indices,
|
||||
model_config=self.model_config,
|
||||
forward_mode=self.forward_mode,
|
||||
out_cache_loc=self.out_cache_loc,
|
||||
|
||||
Reference in New Issue
Block a user