chenchenghao/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

Re-add the API serving timing metrics. (#14744)

Browse Source 
Signed-off-by: zhanghaotong <zhanghaotong.zht@antgroup.com>
Co-authored-by: zhanghaotong <zhanghaotong.zht@antgroup.com>
This commit is contained in:
Liangsheng Yin
2025-12-10 10:17:48 +09:00
committed by GitHub
parent a6dc7d2932
commit cbc7dcdaa7
3 changed files with 37 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
python/sglang/srt/entrypoints/openai/serving_base.py
View File

@@ -12,7 +12,7 @@ from fastapi import HTTPException, Request
from fastapi.responses import ORJSONResponse, StreamingResponse
from sglang.srt.entrypoints.openai.protocol import ErrorResponse, OpenAIServingRequest
from sglang.srt.managers.io_struct import GenerateReqInput
from sglang.srt.managers.io_struct import EmbeddingReqInput, GenerateReqInput
from sglang.srt.server_args import ServerArgs
if TYPE_CHECKING:
@@ -88,6 +88,9 @@ class OpenAIServingBase(ABC):
"""Handle the specific request type with common pattern
If you want to override this method, you should be careful to record the validation time.
"""
received_time = time.time()
received_time_perf = time.perf_counter()
try:
# Validate request
validation_start = time.perf_counter()
@@ -100,8 +103,12 @@ class OpenAIServingBase(ABC):
adapted_request, processed_request = self._convert_to_internal_request(
request, raw_request
)
if hasattr(adapted_request, "validation_time"):
if isinstance(adapted_request, (GenerateReqInput, EmbeddingReqInput)):
# Only set timing fields if adapted_request supports them
adapted_request.validation_time = validation_time
adapted_request.received_time = received_time
adapted_request.received_time_perf = received_time_perf
# Note(Xinyuan): raw_request below is only used for detecting the connection of the client
if hasattr(request, "stream") and request.stream: