From 8359f185a1e4383e1c951d8f1fe291385f7f279a Mon Sep 17 00:00:00 2001 From: zhanghaotong <16302010009@fudan.edu.cn> Date: Wed, 12 Nov 2025 20:28:59 +0800 Subject: [PATCH] [Feature] Propagate Trace Headers into Root Span for OpenTelemetry Cross-Service Context (#10808) Signed-off-by: zhanghaotong --- .../sglang/srt/managers/tokenizer_manager.py | 14 +++++++++--- python/sglang/srt/tracing/trace.py | 22 +++++++++++++++++++ 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/python/sglang/srt/managers/tokenizer_manager.py b/python/sglang/srt/managers/tokenizer_manager.py index 4111e16ac..9c5c6fa12 100644 --- a/python/sglang/srt/managers/tokenizer_manager.py +++ b/python/sglang/srt/managers/tokenizer_manager.py @@ -82,6 +82,7 @@ from sglang.srt.server_args import ( ) from sglang.srt.speculative.spec_info import SpeculativeAlgorithm from sglang.srt.tracing.trace import ( + extract_trace_headers, trace_get_proc_propagate_context, trace_req_finish, trace_req_start, @@ -411,14 +412,18 @@ class TokenizerManager(TokenizerCommunicatorMixin): self.auto_create_handle_loop() obj.normalize_batch_and_arguments() - if request and "trace_context" in request.headers: - trace_set_remote_propagate_context(request.headers["trace_context"]) + external_trace_header = None + if request: + if "trace_context" in request.headers: + trace_set_remote_propagate_context(request.headers["trace_context"]) + else: + external_trace_header = extract_trace_headers(request.headers) if self.server_args.tokenizer_worker_num > 1: self._attach_multi_http_worker_info(obj) if self.enable_trace: - self._trace_request_start(obj, created_time) + self._trace_request_start(obj, created_time, external_trace_header) if self.log_requests: max_length, skip_names, _ = self.log_request_metadata @@ -2301,6 +2306,7 @@ class TokenizerManager(TokenizerCommunicatorMixin): self, obj: Union[GenerateReqInput, EmbeddingReqInput], created_time: Optional[float] = None, + external_trace_header: Optional[Dict] = None, ): if obj.is_single: bootstrap_room = ( @@ -2311,6 +2317,7 @@ class TokenizerManager(TokenizerCommunicatorMixin): bootstrap_room, ts=int(created_time * 1e9), role=self.server_args.disaggregation_mode, + external_trace_header=external_trace_header, ) trace_slice_start("", obj.rid, ts=int(created_time * 1e9), anonymous=True) else: @@ -2325,6 +2332,7 @@ class TokenizerManager(TokenizerCommunicatorMixin): bootstrap_room, ts=int(created_time * 1e9), role=self.server_args.disaggregation_mode, + external_trace_header=external_trace_header, ) trace_slice_start( "", obj.rid[i], ts=int(created_time * 1e9), anonymous=True diff --git a/python/sglang/srt/tracing/trace.py b/python/sglang/srt/tracing/trace.py index e3a12dca7..32d820fc6 100644 --- a/python/sglang/srt/tracing/trace.py +++ b/python/sglang/srt/tracing/trace.py @@ -30,10 +30,14 @@ from sglang.srt.utils import get_int_env_var if TYPE_CHECKING: from sglang.srt.managers.scheduler import Req +from typing import Any, Dict, List, Mapping, Optional logger = logging.getLogger(__name__) opentelemetry_imported = False tracing_enabled = False +_trace_context_propagator = None + +TRACE_HEADERS = ["traceparent", "tracestate"] try: from opentelemetry import context, propagate, trace @@ -49,6 +53,11 @@ try: from opentelemetry.sdk.resources import SERVICE_NAME, Resource from opentelemetry.sdk.trace import TracerProvider, id_generator from opentelemetry.sdk.trace.export import BatchSpanProcessor + from opentelemetry.trace.propagation.tracecontext import ( + TraceContextTextMapPropagator, + ) + + _trace_context_propagator = TraceContextTextMapPropagator() opentelemetry_imported = True except ImportError: @@ -60,6 +69,14 @@ except ImportError: logger.info("opentelemetry package is not installed, tracing disabled") +def is_tracing_enabled() -> bool: + return tracing_enabled + + +def extract_trace_headers(headers: Mapping[str, str]) -> Optional[Dict]: + return {h: headers[h] for h in TRACE_HEADERS if h in headers} + + @dataclass class SglangTraceThreadInfo: host_id: str @@ -418,6 +435,7 @@ def trace_req_start( bootstrap_room: Optional[int] = None, ts: Optional[int] = None, role: Optional[str] = "null", + external_trace_header: Optional[Dict[str, str]] = None, ): if not tracing_enabled: return @@ -444,10 +462,14 @@ def trace_req_start( tracer = threads_info[pid].tracer if str(bootstrap_room) not in remote_trace_contexts: attrs = {"bootstrap_room": str(hex(bootstrap_room))} + external_trace_context = _trace_context_propagator.extract( + external_trace_header + ) bootstrap_room_span = tracer.start_span( name=f"Bootstrap Room {hex(bootstrap_room)}", start_time=ts, attributes=attrs, + context=external_trace_context, ) reqs_context[rid].bootstrap_room_span = bootstrap_room_span bootstrap_room_span_context = trace.set_span_in_context(bootstrap_room_span)