Add timing metrics for requests (#12646)

Co-authored-by: Scott Lee <scottjlee@users.noreply.github.com>
This commit is contained in:
yinghui
2025-11-05 23:07:16 -08:00
committed by GitHub
parent fd3034da75
commit 58095cb00a
9 changed files with 334 additions and 52 deletions

View File

@@ -80,6 +80,10 @@ class GrpcReqState:
last_time: float = 0.0
last_completion_tokens: int = 1
# perf_counter equivalents for accurate time calculations
finished_time_perf: float = 0.0
first_token_time_perf: float = 0.0
# Streaming state
stream_finished: bool = False
input_logprobs_sent: bool = False # Track if input logprobs were sent in streaming
@@ -536,6 +540,7 @@ class GrpcRequestManager:
put_tasks = []
cleanup_tasks = []
now = time.time()
now_perf_counter = time.perf_counter()
# Process each request in the batch
for i, rid in enumerate(batch_out.rids):
@@ -552,6 +557,7 @@ class GrpcRequestManager:
# Update metrics
if state.first_token_time == 0.0:
state.first_token_time = now
state.first_token_time_perf = now_perf_counter
state.last_time = now
# Extract output for this request
@@ -650,6 +656,7 @@ class GrpcRequestManager:
if output_data["finished"]:
state.finished = True
state.finished_time = now
state.finished_time_perf = now_perf_counter
state.stream_finished = True
state.event.set()
@@ -691,6 +698,7 @@ class GrpcRequestManager:
# Mark as finished
state.finished = True
state.finished_time = time.time()
state.finished_time_perf = time.perf_counter()
state.event.set()
async def _handle_health_check_output(self, health_out: HealthCheckOutput):
@@ -723,6 +731,7 @@ class GrpcRequestManager:
# Mark as finished
state.finished = True
state.finished_time = time.time()
state.finished_time_perf = time.perf_counter()
state.event.set()
async def _handle_abort_req(self, recv_obj: AbortReq):