diff --git a/python/sglang/srt/managers/scheduler.py b/python/sglang/srt/managers/scheduler.py index 4f99d8927..cde8f1026 100644 --- a/python/sglang/srt/managers/scheduler.py +++ b/python/sglang/srt/managers/scheduler.py @@ -2055,13 +2055,19 @@ class Scheduler( return batch # Check if decode out of memory - if not batch.check_decode_mem(self.decode_mem_cache_buf_multiplier) or ( - TEST_RETRACT and self.forward_ct % TEST_RETRACT_INTERVAL == 0 - ): + if ( + kv_full_retract_flag := not batch.check_decode_mem( + self.decode_mem_cache_buf_multiplier + ) + ) or (TEST_RETRACT and self.forward_ct % TEST_RETRACT_INTERVAL == 0): + old_available_tokens = self.token_to_kv_pool_allocator.available_size() old_ratio = self.new_token_ratio retracted_reqs, new_token_ratio, reqs_to_abort = batch.retract_decode( self.server_args, self.decode_mem_cache_buf_multiplier ) + new_available_tokens = self.token_to_kv_pool_allocator.available_size() + new_token_gained = new_available_tokens - old_available_tokens + self.num_retracted_reqs = len(retracted_reqs) if self.enable_metrics and (x := len(retracted_reqs)) > 0: self.metrics_collector.increment_num_retracted_reqs(x) @@ -2072,11 +2078,17 @@ class Scheduler( AbortReq(abort_message=abort_reason.message, rid=req.rid), req ) - logger.info( + msg_prefix = ( "KV cache pool is full. Retract requests. " - f"#retracted_reqs: {len(retracted_reqs)}, " - f"#new_token_ratio: {old_ratio:.4f} -> {new_token_ratio:.4f}" + if kv_full_retract_flag + else "Testing retraction. " ) + msg_details = f"#retracted_reqs: {len(retracted_reqs)}, #new_tokens_gained: {new_token_gained}" + if kv_full_retract_flag: + msg_details += ( + f", #new_token_ratio: {old_ratio:.4f} -> {new_token_ratio:.4f}" + ) + logger.warning(msg_prefix + msg_details) for req in retracted_reqs: self._add_request_to_queue(req, is_retracted=True)