[metrics] Add in queue metrics (#4444)

2025-04-09 17:19:27 -07:00
parent 4cb53ecd0c
commit 5db37c8626
3 changed files with 30 additions and 1 deletions
--- a/python/sglang/srt/managers/schedule_batch.py
+++ b/python/sglang/srt/managers/schedule_batch.py
@@ -474,6 +474,10 @@ class Req:
        self.temp_scaled_logprobs = False
        self.top_p_normalized_logprobs = False

+        # Latency Breakdown
+        self.queue_time_start = None
+        self.queue_time_end = None
+
        # Logprobs (return values)
        self.input_token_logprobs_val: Optional[List[float]] = None
        self.input_token_logprobs_idx: Optional[List[int]] = None