From cbc7dcdaa7d638d3a1201e3f4e2e8dade288f692 Mon Sep 17 00:00:00 2001 From: Liangsheng Yin Date: Wed, 10 Dec 2025 10:17:48 +0900 Subject: [PATCH] Re-add the API serving timing metrics. (#14744) Signed-off-by: zhanghaotong Co-authored-by: zhanghaotong --- .../srt/entrypoints/openai/serving_base.py | 11 ++++-- python/sglang/srt/managers/io_struct.py | 34 +++++++++++++++---- .../sglang/srt/managers/tokenizer_manager.py | 2 +- 3 files changed, 37 insertions(+), 10 deletions(-) diff --git a/python/sglang/srt/entrypoints/openai/serving_base.py b/python/sglang/srt/entrypoints/openai/serving_base.py index 669aed7b0..8ee388080 100644 --- a/python/sglang/srt/entrypoints/openai/serving_base.py +++ b/python/sglang/srt/entrypoints/openai/serving_base.py @@ -12,7 +12,7 @@ from fastapi import HTTPException, Request from fastapi.responses import ORJSONResponse, StreamingResponse from sglang.srt.entrypoints.openai.protocol import ErrorResponse, OpenAIServingRequest -from sglang.srt.managers.io_struct import GenerateReqInput +from sglang.srt.managers.io_struct import EmbeddingReqInput, GenerateReqInput from sglang.srt.server_args import ServerArgs if TYPE_CHECKING: @@ -88,6 +88,9 @@ class OpenAIServingBase(ABC): """Handle the specific request type with common pattern If you want to override this method, you should be careful to record the validation time. """ + received_time = time.time() + received_time_perf = time.perf_counter() + try: # Validate request validation_start = time.perf_counter() @@ -100,8 +103,12 @@ class OpenAIServingBase(ABC): adapted_request, processed_request = self._convert_to_internal_request( request, raw_request ) - if hasattr(adapted_request, "validation_time"): + + if isinstance(adapted_request, (GenerateReqInput, EmbeddingReqInput)): + # Only set timing fields if adapted_request supports them adapted_request.validation_time = validation_time + adapted_request.received_time = received_time + adapted_request.received_time_perf = received_time_perf # Note(Xinyuan): raw_request below is only used for detecting the connection of the client if hasattr(request, "stream") and request.stream: diff --git a/python/sglang/srt/managers/io_struct.py b/python/sglang/srt/managers/io_struct.py index b4f9d3335..c7bbc48ee 100644 --- a/python/sglang/srt/managers/io_struct.py +++ b/python/sglang/srt/managers/io_struct.py @@ -110,6 +110,23 @@ class SpeculativeDecodingMetricsMixin: spec_accepted_tokens: List[int] +@dataclass +class APIServingTimingMixin: + # Validation step duration + validation_time: Optional[float] = None + + # For metrics + received_time: Optional[float] = None + + # Perf_counter equivalents for accurate time calculations + received_time_perf: Optional[float] = None + + +_API_SERVING_TIMING_MIXIN_FIELDS = tuple( + APIServingTimingMixin.__dataclass_fields__.keys() +) + + # Parameters for a session @dataclass class SessionParams: @@ -138,7 +155,7 @@ MultimodalDataInputFormat = Union[ @dataclass -class GenerateReqInput(BaseReq): +class GenerateReqInput(BaseReq, APIServingTimingMixin): # The input prompt. It can be a single prompt or a batch of prompts. text: Optional[Union[List[str], str]] = None # The token ids for text; one can specify either text or input_ids @@ -201,9 +218,6 @@ class GenerateReqInput(BaseReq): # For reasoning reasoning: bool = False - # Validation step duration - validation_time: Optional[float] = None - # For data parallel rank routing data_parallel_rank: Optional[int] = None @@ -623,7 +637,6 @@ class GenerateReqInput(BaseReq): decode_tp_size=( self.decode_tp_size[i] if self.decode_tp_size is not None else None ), - validation_time=self.validation_time, data_parallel_rank=( self.data_parallel_rank if self.data_parallel_rank is not None else None ), @@ -635,6 +648,10 @@ class GenerateReqInput(BaseReq): return_bytes=self.return_bytes, return_entropy=self.return_entropy, http_worker_ipc=self.http_worker_ipc, + **{ + field: getattr(self, field) + for field in _API_SERVING_TIMING_MIXIN_FIELDS + }, ) @@ -724,7 +741,7 @@ class BatchTokenizedGenerateReqInput(BaseBatchReq): @dataclass -class EmbeddingReqInput(BaseReq): +class EmbeddingReqInput(BaseReq, APIServingTimingMixin): # The input prompt. It can be a single prompt or a batch of prompts. text: Optional[Union[List[List[str]], List[str], str]] = None # The image input. It can be an image instance, file name, URL, or base64 encoded string. @@ -840,9 +857,12 @@ class EmbeddingReqInput(BaseReq): video_data=self.video_data[i] if self.video_data is not None else None, sampling_params=self.sampling_params[i], rid=self.rid[i], - validation_time=self.validation_time, dimensions=self.dimensions, http_worker_ipc=self.http_worker_ipc, + **{ + field: getattr(self, field) + for field in _API_SERVING_TIMING_MIXIN_FIELDS + }, ) diff --git a/python/sglang/srt/managers/tokenizer_manager.py b/python/sglang/srt/managers/tokenizer_manager.py index 555cfb101..f339d2e9f 100644 --- a/python/sglang/srt/managers/tokenizer_manager.py +++ b/python/sglang/srt/managers/tokenizer_manager.py @@ -408,7 +408,7 @@ class TokenizerManager(TokenizerCommunicatorMixin, TokenizerManagerMultiItemMixi obj: Union[GenerateReqInput, EmbeddingReqInput], request: Optional[fastapi.Request] = None, ): - created_time = time.time() + created_time = obj.received_time if obj.received_time else time.time() self.auto_create_handle_loop() obj.normalize_batch_and_arguments()