[model-gateway][tracing]: implement request tracing using OpenTelemetry with trace context propagation (HTTP) (#13897)

sgl-model-gateway/src/config/builder.rs

@@ -1,7 +1,7 @@
 use super::{
     CircuitBreakerConfig, ConfigError, ConfigResult, DiscoveryConfig, HealthCheckConfig,
     HistoryBackend, MetricsConfig, OracleConfig, PolicyConfig, PostgresConfig, RetryConfig,
-    RouterConfig, RoutingMode, TokenizerCacheConfig,
+    RouterConfig, RoutingMode, TokenizerCacheConfig, TraceConfig,
 };
 use crate::{core::ConnectionMode, mcp::McpConfig};
 
@@ -297,6 +297,24 @@ impl RouterConfigBuilder {
         self
     }
 
+    // ===================== Otel Trace ====================
+
+    pub fn enable_trace<S: Into<String>>(mut self, endpoint: S) -> Self {
+        self.config.trace_config = Some(TraceConfig {
+            enable_trace: true,
+            otlp_traces_endpoint: endpoint.into(),
+        });
+        self
+    }
+
+    pub fn disable_trace(mut self) -> Self {
+        self.config.trace_config = Some(TraceConfig {
+            enable_trace: false,
+            otlp_traces_endpoint: "".to_string(),
+        });
+        self
+    }
+
     // ==================== Logging ====================
 
     pub fn log_dir<S: Into<String>>(mut self, dir: S) -> Self {
@@ -461,6 +479,11 @@ impl RouterConfigBuilder {
         self
     }
 
+    pub fn maybe_trace(mut self, trace_config: Option<TraceConfig>) -> Self {
+        self.config.trace_config = trace_config;
+        self
+    }
+
     pub fn maybe_log_dir(mut self, dir: Option<impl Into<String>>) -> Self {
         self.config.log_dir = dir.map(|d| d.into());
         self
@@ -703,11 +726,13 @@ mod tests {
             .to_builder()
             .port(4000)
             .enable_metrics("0.0.0.0", 29000)
+            .enable_trace("localhost:4317")
             .build()
             .unwrap();
 
         assert_eq!(modified.port, 4000);
         assert!(modified.metrics.is_some());
+        assert!(modified.trace_config.is_some());
     }
 
     /// Test complex routing mode helper method

sgl-model-gateway/src/config/types.rs

@@ -23,6 +23,7 @@ pub struct RouterConfig {
     pub api_key: Option<String>,
     pub discovery: Option<DiscoveryConfig>,
     pub metrics: Option<MetricsConfig>,
+    pub trace_config: Option<TraceConfig>,
     pub log_dir: Option<String>,
     pub log_level: Option<String>,
     pub request_id_headers: Option<Vec<String>>,
@@ -461,6 +462,21 @@ impl Default for MetricsConfig {
     }
 }
 
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct TraceConfig {
+    pub enable_trace: bool,
+    pub otlp_traces_endpoint: String,
+}
+
+impl Default for TraceConfig {
+    fn default() -> Self {
+        Self {
+            enable_trace: false,
+            otlp_traces_endpoint: "localhost:4317".to_string(),
+        }
+    }
+}
+
 impl Default for RouterConfig {
     fn default() -> Self {
         Self {
@@ -478,6 +494,7 @@ impl Default for RouterConfig {
             api_key: None,
             discovery: None,
             metrics: None,
+            trace_config: None,
             log_dir: None,
             log_level: None,
             request_id_headers: None,
@@ -544,6 +561,14 @@ impl RouterConfig {
         self.metrics.is_some()
     }
 
+    /// Check if tracing is enabled
+    pub fn has_tracing(&self) -> bool {
+        match &self.trace_config {
+            Some(trace_config) => trace_config.enable_trace,
+            None => false,
+        }
+    }
+
     /// Compute the effective retry config considering disable flag
     pub fn effective_retry_config(&self) -> RetryConfig {
         let mut cfg = self.retry.clone();
@@ -588,6 +613,7 @@ mod tests {
         assert_eq!(config.worker_startup_check_interval_secs, 30);
         assert!(config.discovery.is_none());
         assert!(config.metrics.is_none());
+        assert!(config.trace_config.is_none());
         assert!(config.log_dir.is_none());
         assert!(config.log_level.is_none());
     }
@@ -636,6 +662,7 @@ mod tests {
         assert_eq!(config.log_level, deserialized.log_level);
         assert!(deserialized.discovery.is_none());
         assert!(deserialized.metrics.is_none());
+        assert!(deserialized.trace_config.is_none());
     }
 
     #[test]
@@ -891,6 +918,25 @@ mod tests {
         assert_eq!(config.host, "0.0.0.0");
     }
 
+    #[test]
+    fn test_trace_config_default() {
+        let config = TraceConfig::default();
+
+        assert!(!config.enable_trace);
+        assert_eq!(config.otlp_traces_endpoint, "localhost:4317");
+    }
+
+    #[test]
+    fn test_trace_config_custom() {
+        let config = TraceConfig {
+            enable_trace: true,
+            otlp_traces_endpoint: "otel-collector:4317".to_string(),
+        };
+
+        assert!(config.enable_trace);
+        assert_eq!(config.otlp_traces_endpoint, "otel-collector:4317");
+    }
+
     #[test]
     fn test_mode_type() {
         let config = RouterConfig::builder()
@@ -932,6 +978,17 @@ mod tests {
         assert!(config.has_metrics());
     }
 
+    #[test]
+    fn test_has_tracing() {
+        let config = RouterConfig::default();
+        assert!(!config.has_tracing());
+
+        let config = RouterConfig::builder()
+            .enable_trace("localhost:4317")
+            .build_unchecked();
+        assert!(config.has_tracing());
+    }
+
     #[test]
     fn test_large_worker_lists() {
         let large_urls: Vec<String> = (0..1000).map(|i| format!("http://worker{}", i)).collect();
@@ -1016,6 +1073,7 @@ mod tests {
                 ..Default::default()
             })
             .enable_metrics("0.0.0.0", 9090)
+            .enable_trace("localhost:4317")
             .log_dir("/var/log/sglang")
             .log_level("info")
             .max_concurrent_requests(64)
@@ -1026,6 +1084,7 @@ mod tests {
         assert_eq!(config.policy.name(), "power_of_two");
         assert!(config.has_service_discovery());
         assert!(config.has_metrics());
+        assert!(config.has_tracing());
     }
 
     #[test]
@@ -1055,6 +1114,7 @@ mod tests {
                 ..Default::default()
             })
             .metrics_config(MetricsConfig::default())
+            .enable_trace("localhost:4317")
             .log_level("debug")
             .max_concurrent_requests(64)
             .build_unchecked();
@@ -1064,6 +1124,7 @@ mod tests {
         assert_eq!(config.policy.name(), "cache_aware");
         assert!(config.has_service_discovery());
         assert!(config.has_metrics());
+        assert!(config.has_tracing());
     }
 
     #[test]
@@ -1092,6 +1153,7 @@ mod tests {
                 bootstrap_port_annotation: "mycompany.io/bootstrap".to_string(),
             })
             .enable_metrics("::", 9999) // IPv6 any
+            .enable_trace("localhost:4317")
             .log_dir("/opt/logs/sglang")
             .log_level("trace")
             .max_concurrent_requests(64)
@@ -1099,6 +1161,7 @@ mod tests {
 
         assert!(config.has_service_discovery());
         assert!(config.has_metrics());
+        assert!(config.has_tracing());
         assert_eq!(config.mode_type(), "regular");
 
         let json = serde_json::to_string_pretty(&config).unwrap();

sgl-model-gateway/src/config/validation.rs

@@ -18,6 +18,10 @@ impl ConfigValidator {
             Self::validate_metrics(metrics)?;
         }
 
+        if let Some(trace_config) = &config.trace_config {
+            Self::validate_trace(trace_config)?;
+        }
+
         Self::validate_compatibility(config)?;
 
         let retry_cfg = config.effective_retry_config();
@@ -357,6 +361,46 @@ impl ConfigValidator {
         Ok(())
     }
 
+    fn validate_trace(trace_config: &TraceConfig) -> ConfigResult<()> {
+        if !trace_config.enable_trace {
+            return Ok(());
+        }
+
+        let endpoint = &trace_config.otlp_traces_endpoint;
+
+        let Some((host, port_str)) = endpoint.rsplit_once(':') else {
+            return Err(ConfigError::InvalidValue {
+                field: "trace_config.otlp_traces_endpoint".to_string(),
+                value: endpoint.clone(),
+                reason:
+                    "expected format <host>:<port>, e.g., otel-collector:4317 or 127.0.0.1:4317"
+                        .to_string(),
+            });
+        };
+
+        if host.is_empty() {
+            return Err(ConfigError::InvalidValue {
+                field: "trace_config.otlp_traces_endpoint".to_string(),
+                value: endpoint.clone(),
+                reason: "host part cannot be empty".to_string(),
+            });
+        }
+
+        // check port: must be 1~65535
+        match port_str.parse::<u16>() {
+            Ok(p) if p > 0 => (), // valid port
+            _ => {
+                return Err(ConfigError::InvalidValue {
+                    field: "trace_config.otlp_traces_endpoint".to_string(),
+                    value: endpoint.clone(),
+                    reason: "port must be a number between 1 and 65535".to_string(),
+                });
+            }
+        };
+
+        Ok(())
+    }
+
     fn validate_retry(retry: &RetryConfig) -> ConfigResult<()> {
         if retry.max_retries < 1 {
             return Err(ConfigError::InvalidValue {

sgl-model-gateway/src/lib.rs

@@ -9,6 +9,7 @@ pub mod mcp;
 pub mod metrics;
 pub mod middleware;
 pub mod multimodal;
+pub mod otel_trace;
 pub mod policies;
 pub mod protocols;
 pub mod reasoning_parser;

sgl-model-gateway/src/logging.rs

@@ -10,6 +10,8 @@ use tracing_subscriber::{
     fmt::time::ChronoUtc, layer::SubscriberExt, util::SubscriberInitExt, EnvFilter, Layer,
 };
 
+use crate::{config::TraceConfig, otel_trace::get_otel_layer};
+
 #[derive(Debug, Clone)]
 pub struct LoggingConfig {
     pub level: Level,
@@ -38,7 +40,7 @@ pub struct LogGuard {
     _file_guard: Option<WorkerGuard>,
 }
 
-pub fn init_logging(config: LoggingConfig) -> LogGuard {
+pub fn init_logging(config: LoggingConfig, otel_layer_config: Option<TraceConfig>) -> LogGuard {
     let _ = LogTracer::init();
 
     let level_filter = match config.level {
@@ -121,6 +123,19 @@ pub fn init_logging(config: LoggingConfig) -> LogGuard {
         layers.push(file_layer);
     }
 
+    if let Some(otel_layer_config) = &otel_layer_config {
+        if otel_layer_config.enable_trace {
+            match get_otel_layer() {
+                Ok(otel_layer) => {
+                    layers.push(otel_layer);
+                }
+                Err(e) => {
+                    eprintln!("Failed to initialize OpenTelemetry: {}", e);
+                }
+            }
+        }
+    }
+
     let _ = tracing_subscriber::registry()
         .with(env_filter)
         .with(layers)

sgl-model-gateway/src/main.rs

@@ -5,10 +5,11 @@ use sgl_model_gateway::{
     config::{
         CircuitBreakerConfig, ConfigError, ConfigResult, DiscoveryConfig, HealthCheckConfig,
         HistoryBackend, MetricsConfig, OracleConfig, PolicyConfig, PostgresConfig, RetryConfig,
-        RouterConfig, RoutingMode, TokenizerCacheConfig,
+        RouterConfig, RoutingMode, TokenizerCacheConfig, TraceConfig,
     },
     core::ConnectionMode,
     metrics::PrometheusConfig,
+    otel_trace::{is_otel_enabled, shutdown_otel},
     server::{self, ServerConfig},
     service_discovery::ServiceDiscoveryConfig,
     version,
@@ -348,6 +349,12 @@ struct CliArgs {
 
     #[arg(long, default_value_t = false)]
     enable_wasm: bool,
+
+    #[arg(long, default_value_t = false)]
+    enable_trace: bool,
+
+    #[arg(long, default_value = "localhost:4317")]
+    otlp_traces_endpoint: String,
 }
 
 enum OracleConnectSource {
@@ -539,6 +546,11 @@ impl CliArgs {
             host: self.prometheus_host.clone(),
         });
 
+        let trace_config = Some(TraceConfig {
+            enable_trace: self.enable_trace,
+            otlp_traces_endpoint: self.otlp_traces_endpoint.clone(),
+        });
+
         let mut all_urls = Vec::new();
         match &mode {
             RoutingMode::Regular { worker_urls } => {
@@ -624,6 +636,7 @@ impl CliArgs {
             .maybe_api_key(self.api_key.as_ref())
             .maybe_discovery(discovery)
             .maybe_metrics(metrics)
+            .maybe_trace(trace_config)
             .maybe_log_dir(self.log_dir.as_ref())
             .maybe_request_id_headers(
                 (!self.request_id_headers.is_empty()).then(|| self.request_id_headers.clone()),
@@ -768,6 +781,8 @@ Provide --worker-urls or PD flags as usual.",
     let server_config = cli_args.to_server_config(router_config);
     let runtime = tokio::runtime::Runtime::new()?;
     runtime.block_on(async move { server::startup(server_config).await })?;
-
+    if is_otel_enabled() {
+        shutdown_otel();
+    }
     Ok(())
 }

sgl-model-gateway/src/middleware.rs

@@ -207,6 +207,7 @@ impl<B> MakeSpan<B> for RequestSpan {
         // Don't try to extract request ID here - it won't be available yet
         // The RequestIdLayer runs after TraceLayer creates the span
         info_span!(
+            target: "sgl_model_gateway::otel-trace",
             "http_request",
             method = %request.method(),
             uri = %request.uri(),
@@ -215,6 +216,7 @@ impl<B> MakeSpan<B> for RequestSpan {
             status_code = Empty,
             latency = Empty,
             error = Empty,
+            module = "sglang::router_rs"
         )
     }
 }

sgl-model-gateway/src/otel_trace.rs

@@ -0,0 +1,229 @@
+use std::{
+    collections::HashSet,
+    sync::{
+        atomic::{AtomicBool, Ordering},
+        OnceLock,
+    },
+    time::Duration,
+};
+
+use anyhow::Result;
+use axum::http::{HeaderMap, HeaderName, HeaderValue};
+use opentelemetry::{global, trace::TracerProvider as _, KeyValue};
+use opentelemetry_otlp::WithExportConfig;
+use opentelemetry_sdk::{
+    propagation::TraceContextPropagator,
+    runtime,
+    trace::{BatchConfigBuilder, BatchSpanProcessor, Tracer as SdkTracer, TracerProvider},
+    Resource,
+};
+use tokio::task::spawn_blocking;
+use tracing::{Metadata, Subscriber};
+use tracing_opentelemetry::{self, OpenTelemetrySpanExt};
+use tracing_subscriber::{
+    layer::{Context, Filter},
+    Layer,
+};
+
+use crate::routers::http::events::get_module_path as http_router_get_module_path;
+
+static ENABLED: AtomicBool = AtomicBool::new(false);
+
+// global tracer
+static TRACER: OnceLock<SdkTracer> = OnceLock::new();
+static PROVIDER: OnceLock<TracerProvider> = OnceLock::new();
+
+pub struct CustomOtelFilter {
+    allowed_targets: HashSet<String>,
+}
+
+impl CustomOtelFilter {
+    pub fn new() -> Self {
+        let mut allowed_targets = HashSet::new();
+        allowed_targets.insert("sgl_model_gateway::otel-trace".to_string());
+        allowed_targets.insert(http_router_get_module_path().to_string());
+
+        Self { allowed_targets }
+    }
+}
+
+impl<S> Filter<S> for CustomOtelFilter
+where
+    S: Subscriber,
+{
+    fn enabled(&self, meta: &Metadata<'_>, _cx: &Context<'_, S>) -> bool {
+        self.allowed_targets.contains(meta.target())
+    }
+
+    fn callsite_enabled(&self, meta: &'static Metadata<'static>) -> tracing::subscriber::Interest {
+        if self.allowed_targets.contains(meta.target()) {
+            tracing::subscriber::Interest::always()
+        } else {
+            tracing::subscriber::Interest::never()
+        }
+    }
+}
+
+impl Default for CustomOtelFilter {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+/// init OpenTelemetry connection
+pub fn otel_tracing_init(enable: bool, otlp_endpoint: Option<&str>) -> Result<()> {
+    if !enable {
+        ENABLED.store(false, Ordering::Relaxed);
+        return Ok(());
+    }
+
+    let endpoint = otlp_endpoint.unwrap_or("localhost:4317");
+    let endpoint = if !endpoint.starts_with("http://") && !endpoint.starts_with("https://") {
+        format!("http://{}", endpoint)
+    } else {
+        endpoint.to_string()
+    };
+
+    let result = std::panic::catch_unwind(|| -> Result<()> {
+        global::set_text_map_propagator(TraceContextPropagator::new());
+
+        let exporter = opentelemetry_otlp::SpanExporter::builder()
+            .with_tonic()
+            .with_endpoint(endpoint)
+            .with_protocol(opentelemetry_otlp::Protocol::Grpc)
+            .build()?;
+
+        let batch_config = BatchConfigBuilder::default()
+            .with_scheduled_delay(Duration::from_millis(500))
+            .with_max_export_batch_size(64)
+            .build();
+
+        let span_processor = BatchSpanProcessor::builder(exporter, runtime::Tokio)
+            .with_batch_config(batch_config)
+            .build();
+
+        let resource = Resource::default().merge(&Resource::new(vec![KeyValue::new(
+            "service.name",
+            "sgl-router",
+        )]));
+
+        let provider = TracerProvider::builder()
+            .with_span_processor(span_processor)
+            .with_resource(resource)
+            .build();
+        PROVIDER
+            .set(provider.clone())
+            .map_err(|_| anyhow::anyhow!("Provider already initialized"))?;
+
+        let tracer = provider.tracer("sgl-router");
+
+        TRACER
+            .set(tracer)
+            .map_err(|_| anyhow::anyhow!("Tracer already initialized"))?;
+
+        let _ = global::set_tracer_provider(provider);
+
+        ENABLED.store(true, Ordering::Relaxed);
+
+        Ok(())
+    });
+
+    match result {
+        Ok(Ok(())) => {
+            eprintln!(
+                "[tracing] OpenTelemetry initialized successfully, enabled: {}",
+                ENABLED.load(Ordering::Relaxed)
+            );
+            Ok(())
+        }
+        Ok(Err(e)) => {
+            eprintln!("[tracing] Failed to initialize OTLP tracer: {}", e);
+            ENABLED.store(false, Ordering::Relaxed);
+            Err(e)
+        }
+        Err(_) => {
+            eprintln!("[tracing] Panic during OpenTelemetry initialization");
+            ENABLED.store(false, Ordering::Relaxed);
+            Err(anyhow::anyhow!("Panic during initialization"))
+        }
+    }
+}
+
+pub fn get_otel_layer<S>() -> Result<Box<dyn Layer<S> + Send + Sync + 'static>, &'static str>
+where
+    S: Subscriber + for<'a> tracing_subscriber::registry::LookupSpan<'a> + Send + Sync,
+{
+    if !is_otel_enabled() {
+        return Err("OpenTelemetry is not enabled");
+    }
+
+    let tracer = TRACER
+        .get()
+        .ok_or("Tracer not initialized. Call otel_tracing_init first.")?
+        .clone();
+
+    let custom_filter = CustomOtelFilter::new();
+
+    let layer = tracing_opentelemetry::layer()
+        .with_tracer(tracer)
+        .with_filter(custom_filter);
+
+    Ok(Box::new(layer))
+}
+
+pub fn is_otel_enabled() -> bool {
+    ENABLED.load(Ordering::Relaxed)
+}
+
+pub async fn flush_spans_async() -> Result<()> {
+    if !is_otel_enabled() {
+        return Ok(());
+    }
+
+    if let Some(provider) = PROVIDER.get() {
+        let provider = provider.clone();
+
+        spawn_blocking(move || provider.force_flush())
+            .await
+            .map_err(|e| {
+                anyhow::anyhow!("Failed to join blocking task for flushing spans: {}", e)
+            })?;
+
+        Ok(())
+    } else {
+        Err(anyhow::anyhow!("Provider not initialized"))
+    }
+}
+
+pub fn shutdown_otel() {
+    if ENABLED.load(Ordering::Relaxed) {
+        global::shutdown_tracer_provider();
+        ENABLED.store(false, Ordering::Relaxed);
+        eprintln!("[tracing] OpenTelemetry shut down");
+    }
+}
+
+pub fn inject_trace_context_http(headers: &mut HeaderMap) -> Result<()> {
+    if !is_otel_enabled() {
+        return Err(anyhow::anyhow!("OTEL not enabled"));
+    }
+
+    let context = tracing::Span::current().context();
+
+    struct HeaderInjector<'a>(&'a mut HeaderMap);
+
+    impl<'a> opentelemetry::propagation::Injector for HeaderInjector<'a> {
+        fn set(&mut self, key: &str, value: String) {
+            if let Ok(header_name) = HeaderName::from_bytes(key.as_bytes()) {
+                if let Ok(header_value) = HeaderValue::from_str(&value) {
+                    self.0.insert(header_name, header_value);
+                }
+            }
+        }
+    }
+
+    global::get_text_map_propagator(|propagator| {
+        propagator.inject_context(&context, &mut HeaderInjector(headers));
+    });
+    Ok(())
+}

sgl-model-gateway/src/routers/http/events.rs

@@ -0,0 +1,69 @@
+//! request events for observability and monitoring
+
+use tracing::{debug, event, Level};
+
+use crate::otel_trace::is_otel_enabled;
+
+pub fn get_module_path() -> &'static str {
+    module_path!()
+}
+
+pub trait Event {
+    fn emit(&self);
+}
+
+#[derive(Debug)]
+pub struct RequestPDSentEvent {
+    pub prefill_url: String,
+    pub decode_url: String,
+}
+
+impl Event for RequestPDSentEvent {
+    fn emit(&self) {
+        if !is_otel_enabled() {
+            debug!(
+                "Sending concurrent requests to prefill={} decode={}",
+                self.prefill_url, self.decode_url
+            );
+        } else {
+            event!(
+                Level::INFO,
+                prefill_url = %self.prefill_url,
+                decode_url = %self.decode_url,
+                "Sending concurrent requests"
+            );
+        }
+    }
+}
+
+#[derive(Debug)]
+pub struct RequestSentEvent {
+    pub url: String,
+}
+
+impl Event for RequestSentEvent {
+    fn emit(&self) {
+        if !is_otel_enabled() {
+            debug!("Sending request to {}", self.url);
+        } else {
+            event!(
+                Level::INFO,
+                url = %self.url,
+                "Sending requests"
+            );
+        }
+    }
+}
+
+#[derive(Debug)]
+pub struct RequestReceivedEvent {}
+
+impl Event for RequestReceivedEvent {
+    fn emit(&self) {
+        if !is_otel_enabled() {
+            debug!("Received concurrent requests");
+        } else {
+            event!(Level::INFO, "Received concurrent requests");
+        }
+    }
+}

sgl-model-gateway/src/routers/http/mod.rs

@@ -1,5 +1,6 @@
 //! HTTP router implementations
 
+pub mod events;
 pub mod pd_router;
 pub mod pd_types;
 pub mod router;

sgl-model-gateway/src/routers/http/pd_router.rs

@@ -14,13 +14,17 @@ use serde_json::{json, Value};
 use tokio_stream::wrappers::UnboundedReceiverStream;
 use tracing::{debug, error, warn};
 
-use super::pd_types::api_path;
+use super::{
+    events::{self, Event},
+    pd_types::api_path,
+};
 use crate::{
     config::types::RetryConfig,
     core::{
         is_retryable_status, RetryExecutor, Worker, WorkerLoadGuard, WorkerRegistry, WorkerType,
     },
     metrics::RouterMetrics,
+    otel_trace::inject_trace_context_http,
     policies::{LoadBalancingPolicy, PolicyRegistry},
     protocols::{
         chat::{ChatCompletionRequest, ChatMessage, MessageContent},
@@ -391,6 +395,12 @@ impl PDRouter {
             None
         };
 
+        let mut headers_with_trace = headers.cloned().unwrap_or_default();
+        let headers = match inject_trace_context_http(&mut headers_with_trace) {
+            Ok(()) => Some(&headers_with_trace),
+            Err(_) => headers,
+        };
+
         // Build both requests
         let prefill_request = self.build_post_with_headers(
             &self.client,
@@ -410,15 +420,16 @@ impl PDRouter {
         );
 
         // Send both requests concurrently and wait for both
-        debug!(
-            "Sending concurrent requests to prefill={} decode={}",
-            prefill.url(),
-            decode.url()
-        );
+        events::RequestPDSentEvent {
+            prefill_url: prefill.url().to_string(),
+            decode_url: decode.url().to_string(),
+        }
+        .emit();
 
         let (prefill_result, decode_result) =
             tokio::join!(prefill_request.send(), decode_request.send());
-        debug!("Received responses from both servers");
+
+        events::RequestReceivedEvent {}.emit();
 
         let duration = start_time.elapsed();
         RouterMetrics::record_pd_request_duration(context.route, duration);
@@ -873,7 +884,11 @@ impl PDRouter {
                 // Whitelist important end-to-end headers, skip hop-by-hop
                 let forward = matches!(
                     name_lc.as_str(),
-                    "authorization" | "x-request-id" | "x-correlation-id"
+                    "authorization"
+                    | "x-request-id"
+                    | "x-correlation-id"
+                    | "traceparent"      // W3C Trace Context
+                    | "tracestate" // W3C Trace Context
                 ) || name_lc.starts_with("x-request-id-");
                 if forward {
                     if let Ok(val) = value.to_str() {

sgl-model-gateway/src/routers/http/router.rs

@@ -15,12 +15,14 @@ use reqwest::Client;
 use tokio_stream::wrappers::UnboundedReceiverStream;
 use tracing::{debug, error};
 
+use super::events::{self, Event};
 use crate::{
     config::types::RetryConfig,
     core::{
         is_retryable_status, ConnectionMode, RetryExecutor, Worker, WorkerRegistry, WorkerType,
     },
     metrics::RouterMetrics,
+    otel_trace::inject_trace_context_http,
     policies::PolicyRegistry,
     protocols::{
         chat::ChatCompletionRequest,
@@ -212,6 +214,16 @@ impl Router {
                     None
                 };
 
+                events::RequestSentEvent {
+                    url: worker.url().to_string(),
+                }
+                .emit();
+                let mut headers_with_trace = headers.cloned().unwrap_or_default();
+                let headers = match inject_trace_context_http(&mut headers_with_trace) {
+                    Ok(()) => Some(&headers_with_trace),
+                    Err(_) => headers,
+                };
+
                 let response = self
                     .send_typed_request(
                         headers,
@@ -223,6 +235,8 @@ impl Router {
                     )
                     .await;
 
+                events::RequestReceivedEvent {}.emit();
+
                 worker.record_outcome(response.status().is_success());
 
                 // For retryable failures, we need to decrement load since send_typed_request

sgl-model-gateway/src/server.rs

@@ -35,6 +35,7 @@ use crate::{
     logging::{self, LoggingConfig},
     metrics::{self, PrometheusConfig},
     middleware::{self, AuthConfig, QueuedRequest},
+    otel_trace,
     protocols::{
         chat::ChatCompletionRequest,
         classify::ClassifyRequest,
@@ -703,25 +704,35 @@ pub fn build_app(
 pub async fn startup(config: ServerConfig) -> Result<(), Box<dyn std::error::Error>> {
     static LOGGING_INITIALIZED: AtomicBool = AtomicBool::new(false);
 
+    if let Some(trace_config) = &config.router_config.trace_config {
+        otel_trace::otel_tracing_init(
+            trace_config.enable_trace,
+            Some(&trace_config.otlp_traces_endpoint),
+        )?;
+    }
+
     let _log_guard = if !LOGGING_INITIALIZED.swap(true, Ordering::SeqCst) {
-        Some(logging::init_logging(LoggingConfig {
-            level: config
-                .log_level
-                .as_deref()
-                .and_then(|s| match s.to_uppercase().parse::<Level>() {
-                    Ok(l) => Some(l),
-                    Err(_) => {
-                        warn!("Invalid log level string: '{s}'. Defaulting to INFO.");
-                        None
-                    }
-                })
-                .unwrap_or(Level::INFO),
-            json_format: false,
-            log_dir: config.log_dir.clone(),
-            colorize: true,
-            log_file_name: "sgl-model-gateway".to_string(),
-            log_targets: None,
-        }))
+        Some(logging::init_logging(
+            LoggingConfig {
+                level: config
+                    .log_level
+                    .as_deref()
+                    .and_then(|s| match s.to_uppercase().parse::<Level>() {
+                        Ok(l) => Some(l),
+                        Err(_) => {
+                            warn!("Invalid log level string: '{s}'. Defaulting to INFO.");
+                            None
+                        }
+                    })
+                    .unwrap_or(Level::INFO),
+                json_format: false,
+                log_dir: config.log_dir.clone(),
+                colorize: true,
+                log_file_name: "sgl-model-gateway".to_string(),
+                log_targets: None,
+            },
+            config.router_config.trace_config.clone(),
+        ))
     } else {
         None
     };