Tiny add more information in retract logging. (#15694)
This commit is contained in:
@@ -2055,13 +2055,19 @@ class Scheduler(
|
||||
return batch
|
||||
|
||||
# Check if decode out of memory
|
||||
if not batch.check_decode_mem(self.decode_mem_cache_buf_multiplier) or (
|
||||
TEST_RETRACT and self.forward_ct % TEST_RETRACT_INTERVAL == 0
|
||||
):
|
||||
if (
|
||||
kv_full_retract_flag := not batch.check_decode_mem(
|
||||
self.decode_mem_cache_buf_multiplier
|
||||
)
|
||||
) or (TEST_RETRACT and self.forward_ct % TEST_RETRACT_INTERVAL == 0):
|
||||
old_available_tokens = self.token_to_kv_pool_allocator.available_size()
|
||||
old_ratio = self.new_token_ratio
|
||||
retracted_reqs, new_token_ratio, reqs_to_abort = batch.retract_decode(
|
||||
self.server_args, self.decode_mem_cache_buf_multiplier
|
||||
)
|
||||
new_available_tokens = self.token_to_kv_pool_allocator.available_size()
|
||||
new_token_gained = new_available_tokens - old_available_tokens
|
||||
|
||||
self.num_retracted_reqs = len(retracted_reqs)
|
||||
if self.enable_metrics and (x := len(retracted_reqs)) > 0:
|
||||
self.metrics_collector.increment_num_retracted_reqs(x)
|
||||
@@ -2072,11 +2078,17 @@ class Scheduler(
|
||||
AbortReq(abort_message=abort_reason.message, rid=req.rid), req
|
||||
)
|
||||
|
||||
logger.info(
|
||||
msg_prefix = (
|
||||
"KV cache pool is full. Retract requests. "
|
||||
f"#retracted_reqs: {len(retracted_reqs)}, "
|
||||
f"#new_token_ratio: {old_ratio:.4f} -> {new_token_ratio:.4f}"
|
||||
if kv_full_retract_flag
|
||||
else "Testing retraction. "
|
||||
)
|
||||
msg_details = f"#retracted_reqs: {len(retracted_reqs)}, #new_tokens_gained: {new_token_gained}"
|
||||
if kv_full_retract_flag:
|
||||
msg_details += (
|
||||
f", #new_token_ratio: {old_ratio:.4f} -> {new_token_ratio:.4f}"
|
||||
)
|
||||
logger.warning(msg_prefix + msg_details)
|
||||
|
||||
for req in retracted_reqs:
|
||||
self._add_request_to_queue(req, is_retracted=True)
|
||||
|
||||
Reference in New Issue
Block a user