From c8dc543dc596107ecd5b5eeca4c752c48cb27c4b Mon Sep 17 00:00:00 2001 From: Haotong Zhang Date: Fri, 30 Jan 2026 16:02:05 +0800 Subject: [PATCH] SGLang Tracing: Improve root span attributes (#17008) Signed-off-by: zhanghaotong --- .../sglang/srt/managers/tokenizer_manager.py | 100 +++++++++++++++++- python/sglang/srt/tracing/trace.py | 22 ++++ 2 files changed, 121 insertions(+), 1 deletion(-) diff --git a/python/sglang/srt/managers/tokenizer_manager.py b/python/sglang/srt/managers/tokenizer_manager.py index 81ad90e6b..3ba650cba 100644 --- a/python/sglang/srt/managers/tokenizer_manager.py +++ b/python/sglang/srt/managers/tokenizer_manager.py @@ -16,6 +16,7 @@ import asyncio import copy import dataclasses +import json import logging import os import pickle @@ -90,6 +91,7 @@ from sglang.srt.server_args import ( ) from sglang.srt.speculative.spec_info import SpeculativeAlgorithm from sglang.srt.tracing.trace import ( + SpanAttributes, extract_trace_headers, trace_get_proc_propagate_context, trace_req_finish, @@ -1593,7 +1595,11 @@ class TokenizerManager(TokenizerCommunicatorMixin, TokenizerManagerMultiItemMixi if self.enable_metrics: self._calculate_timing_metrics(meta_info, state, recv_obj, i) - trace_req_finish(rid, ts=int(state.finished_time * 1e9)) + trace_req_finish( + rid, + ts=int(state.finished_time * 1e9), + attrs=self.convert_to_span_attrs(state, recv_obj, i), + ) del self.rid_to_state[rid] @@ -2284,6 +2290,98 @@ class TokenizerManager(TokenizerCommunicatorMixin, TokenizerManagerMultiItemMixi ): self.mm_receiver.send_encode_request(obj) + def convert_to_span_attrs( + self, + state: ReqState, + recv_obj: Union[ + BatchStrOutput, + BatchEmbeddingOutput, + BatchMultimodalOutput, + BatchTokenIDOutput, + ], + i: int, + ) -> Dict[str, Any]: + """Convert attributes to span attributes.""" + span_attrs = {} + + if not self.enable_trace: + return span_attrs + + # Token usage attributes + span_attrs[SpanAttributes.GEN_AI_USAGE_COMPLETION_TOKENS] = ( + recv_obj.completion_tokens[i] + ) + span_attrs[SpanAttributes.GEN_AI_USAGE_PROMPT_TOKENS] = recv_obj.prompt_tokens[ + i + ] + span_attrs[SpanAttributes.GEN_AI_USAGE_CACHED_TOKENS] = recv_obj.cached_tokens[ + i + ] + + # Request identifiers + span_attrs[SpanAttributes.GEN_AI_REQUEST_ID] = ( + str(state.obj.rid) if state.obj.rid else None + ) + + # Sampling parameters + sampling_params = state.obj.sampling_params or {} + + if max_new_tokens := sampling_params.get("max_new_tokens"): + span_attrs[SpanAttributes.GEN_AI_REQUEST_MAX_TOKENS] = max_new_tokens + + if top_p := sampling_params.get("top_p"): + span_attrs[SpanAttributes.GEN_AI_REQUEST_TOP_P] = top_p + + if temperature := sampling_params.get("temperature"): + span_attrs[SpanAttributes.GEN_AI_REQUEST_TEMPERATURE] = temperature + + if top_k := sampling_params.get("top_k"): + span_attrs[SpanAttributes.GEN_AI_REQUEST_TOP_K] = top_k + + if n := sampling_params.get("n"): + span_attrs[SpanAttributes.GEN_AI_REQUEST_N] = n + + # Response attributes + span_attrs[SpanAttributes.GEN_AI_RESPONSE_MODEL] = self.served_model_name + + finish_reason = ( + recv_obj.finished_reasons[i].get("type") + if recv_obj.finished_reasons[i] + else None + ) + if finish_reason: + span_attrs[SpanAttributes.GEN_AI_RESPONSE_FINISH_REASONS] = json.dumps( + [finish_reason] + ) + + # Latency attributes + if state.first_token_time and state.created_time: + span_attrs[SpanAttributes.GEN_AI_LATENCY_TIME_TO_FIRST_TOKEN] = ( + state.first_token_time - state.created_time + ) + + if state.finished_time and state.created_time: + span_attrs[SpanAttributes.GEN_AI_LATENCY_E2E] = ( + state.finished_time - state.created_time + ) + + if state.first_token_time_perf and state.finished_time_perf: + span_attrs[SpanAttributes.GEN_AI_LATENCY_TIME_IN_MODEL_DECODE] = ( + state.finished_time_perf - state.first_token_time_perf + ) + + if state.request_sent_to_scheduler_ts and state.finished_time: + span_attrs[SpanAttributes.GEN_AI_LATENCY_TIME_IN_MODEL_INFERENCE] = ( + state.finished_time - state.request_sent_to_scheduler_ts + ) + + if state.request_sent_to_scheduler_ts and state.first_token_time: + span_attrs[SpanAttributes.GEN_AI_LATENCY_TIME_IN_MODEL_PREFILL] = ( + state.first_token_time - state.request_sent_to_scheduler_ts + ) + + return span_attrs + class ServerStatus(Enum): Up = "Up" diff --git a/python/sglang/srt/tracing/trace.py b/python/sglang/srt/tracing/trace.py index 355c477b0..6ae1c8738 100644 --- a/python/sglang/srt/tracing/trace.py +++ b/python/sglang/srt/tracing/trace.py @@ -737,3 +737,25 @@ def trace_event_batch( for req in reqs: trace_event(name, req.rid, ts=ts, attrs=_attrs) + + +class SpanAttributes: + # Attribute names copied from here to avoid version conflicts: + # https://github.com/open-telemetry/semantic-conventions/blob/main/docs/gen-ai/gen-ai-spans.md + GEN_AI_USAGE_COMPLETION_TOKENS = "gen_ai.usage.completion_tokens" + GEN_AI_USAGE_PROMPT_TOKENS = "gen_ai.usage.prompt_tokens" + GEN_AI_USAGE_CACHED_TOKENS = "gen_ai.usage.cached_tokens" + GEN_AI_REQUEST_MAX_TOKENS = "gen_ai.request.max_tokens" + GEN_AI_REQUEST_TOP_P = "gen_ai.request.top_p" + GEN_AI_REQUEST_TOP_K = "gen_ai.request.top_k" + GEN_AI_REQUEST_TEMPERATURE = "gen_ai.request.temperature" + GEN_AI_RESPONSE_MODEL = "gen_ai.response.model" + GEN_AI_RESPONSE_FINISH_REASONS = "gen_ai.response.finish_reasons" + GEN_AI_REQUEST_ID = "gen_ai.request.id" + GEN_AI_REQUEST_N = "gen_ai.request.n" + GEN_AI_LATENCY_TIME_IN_QUEUE = "gen_ai.latency.time_in_queue" + GEN_AI_LATENCY_TIME_TO_FIRST_TOKEN = "gen_ai.latency.time_to_first_token" + GEN_AI_LATENCY_E2E = "gen_ai.latency.e2e" + GEN_AI_LATENCY_TIME_IN_MODEL_PREFILL = "gen_ai.latency.time_in_model_prefill" + GEN_AI_LATENCY_TIME_IN_MODEL_DECODE = "gen_ai.latency.time_in_model_decode" + GEN_AI_LATENCY_TIME_IN_MODEL_INFERENCE = "gen_ai.latency.time_in_model_inference"