use admin_api::request_context::{TRACE_ID_HEADER, apply_request_context}; use axum::{ Router, body::Body, http::Request, response::Response, routing::{MethodRouter, get}, }; use opentelemetry::{ Context, trace::{SpanId, TraceContextExt, TracerProvider as _}, }; use opentelemetry_sdk::{ error::OTelSdkResult, trace::{SdkTracerProvider, SpanData, SpanExporter}, }; use std::sync::{Arc, Mutex}; use tower::ServiceExt; use tracing::Instrument; use tracing_subscriber::layer::SubscriberExt; const REMOTE_TRACE_ID: &str = "0af7651916cd43dd8448eb211c80319c"; const REMOTE_PARENT_SPAN_ID: &str = "b7ad6b7169203331"; #[derive(Clone, Debug, Default)] struct RecordingSpanExporter { spans: Arc>>, } impl RecordingSpanExporter { fn finished(&self) -> Vec { self.spans .lock() .expect("recording exporter lock must be available") .clone() } fn named(&self, name: &str) -> Option { self.finished() .into_iter() .find(|span| span.name.as_ref() == name) } } impl SpanExporter for RecordingSpanExporter { async fn export(&self, batch: Vec) -> OTelSdkResult { self.spans .lock() .expect("recording exporter lock must be available") .extend(batch); Ok(()) } } struct TracedRequest { app: Router, provider: SdkTracerProvider, exporter: RecordingSpanExporter, _dispatch_guard: tracing::dispatcher::DefaultGuard, } impl TracedRequest { fn new(route: MethodRouter) -> Self { let exporter = RecordingSpanExporter::default(); let provider = SdkTracerProvider::builder() .with_simple_exporter(exporter.clone()) .build(); let tracer = provider.tracer("admin-request-context-parent-test"); let subscriber = tracing_subscriber::registry().with(tracing_opentelemetry::layer().with_tracer(tracer)); let dispatch = tracing::Dispatch::new(subscriber); let dispatch_guard = tracing::dispatcher::set_default(&dispatch); let app = Router::new() .route("/probe", route) .layer(axum::middleware::from_fn(apply_request_context)); Self { app, provider, exporter, _dispatch_guard: dispatch_guard, } } async fn send(&self, headers: &[(&str, &str)]) -> Response { let mut request = Request::builder().uri("/probe"); for (name, value) in headers { request = request.header(*name, *value); } self.app .clone() .oneshot(request.body(Body::empty()).unwrap()) .await .expect("probe request must succeed") } fn response_trace_id(response: &Response) -> String { response.headers()[TRACE_ID_HEADER] .to_str() .expect("response trace id header must be visible ascii") .to_owned() } fn shutdown(self) { let Self { provider, _dispatch_guard, .. } = self; let _ = provider.shutdown(); } } async fn probe() -> &'static str { "ok" } async fn stage_probe() -> &'static str { let span = crank_trace::Stage::AuthResolve.span(); async { crank_trace::StageOutcome::Success.record(&span); } .instrument(span.clone()) .await; "ok" } fn assert_request_span_is_exported(request: &TracedRequest, expected_trace_id: &str) { let span = request.exporter.named("http.request").unwrap_or_else(|| { panic!( "inbound http.request span must be recorded and exported; exported spans: {:?}", request .exporter .finished() .iter() .map(|span| span.name.to_string()) .collect::>() ) }); assert!( span.span_context.is_valid(), "inbound http.request span must carry a valid span context" ); assert!( span.span_context.is_sampled(), "inbound http.request span must be sampled" ); assert_eq!( span.span_context.trace_id().to_string(), expected_trace_id, "exported request span must carry the trace id reported to the client" ); } #[tokio::test(flavor = "current_thread")] async fn preserves_remote_trace_id_with_an_active_tracer_and_no_global_propagator() { let provider = SdkTracerProvider::builder().build(); let tracer = provider.tracer("admin-request-context-parent-test"); let subscriber = tracing_subscriber::registry().with(tracing_opentelemetry::layer().with_tracer(tracer)); let dispatch = tracing::Dispatch::new(subscriber); let _dispatch_guard = tracing::dispatcher::set_default(&dispatch); let app = Router::new() .route("/probe", get(probe)) .layer(axum::middleware::from_fn(apply_request_context)); let response = app .oneshot( Request::builder() .uri("/probe") .header( "traceparent", "00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01", ) .body(Body::empty()) .unwrap(), ) .await .unwrap(); assert_eq!( response.headers()[TRACE_ID_HEADER], "0af7651916cd43dd8448eb211c80319c" ); provider.shutdown().unwrap(); } #[tokio::test(flavor = "current_thread")] async fn exports_the_inbound_request_span_when_the_ambient_otel_context_is_suppressed() { let request = TracedRequest::new(get(probe)); // A suppressed ambient OTel context is exactly what the intermittent production gate failure // shows: a contextual `crank::trace` span inherits the suppression, is created non-recording // and therefore never reaches the exporter. let _suppressed = Context::enter_telemetry_suppressed_scope(); let response = request.send(&[]).await; assert_eq!(response.status(), 200); let trace_id = TracedRequest::response_trace_id(&response); assert_request_span_is_exported(&request, &trace_id); request.shutdown(); } #[tokio::test(flavor = "current_thread")] async fn exports_the_inbound_request_span_when_an_unrelated_span_is_the_ambient_context() { let request = TracedRequest::new(get(probe)); let ambient_tracer = request.provider.tracer("unrelated-ambient"); let ambient_span = opentelemetry::trace::Tracer::start(&ambient_tracer, "unrelated.stage"); let ambient_trace_id = opentelemetry::trace::Span::span_context(&ambient_span) .trace_id() .to_string(); let _ambient_guard = Context::current_with_span(ambient_span).attach(); let response = request.send(&[]).await; assert_eq!(response.status(), 200); let trace_id = TracedRequest::response_trace_id(&response); assert_request_span_is_exported(&request, &trace_id); let span = request.exporter.named("http.request").unwrap(); assert_ne!( span.span_context.trace_id().to_string(), ambient_trace_id, "an inbound request without traceparent must not join an unrelated ambient trace" ); assert!( span.parent_span_id == SpanId::INVALID, "an inbound request without traceparent must be a root span" ); request.shutdown(); } #[tokio::test(flavor = "current_thread")] async fn preserves_remote_parent_identity_and_sampling_decision() { let sampled = TracedRequest::new(get(probe)); let response = sampled .send(&[( "traceparent", "00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01", )]) .await; assert_eq!(TracedRequest::response_trace_id(&response), REMOTE_TRACE_ID); let span = sampled .exporter .named("http.request") .expect("sampled remote parent must be exported"); assert_eq!(span.span_context.trace_id().to_string(), REMOTE_TRACE_ID); assert_eq!( span.parent_span_id.to_string(), REMOTE_PARENT_SPAN_ID, "remote traceparent parent span must be preserved" ); assert!(span.span_context.is_sampled()); sampled.shutdown(); let unsampled = TracedRequest::new(get(probe)); let response = unsampled .send(&[( "traceparent", "00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-00", )]) .await; assert_eq!(TracedRequest::response_trace_id(&response), REMOTE_TRACE_ID); assert!( unsampled.exporter.finished().is_empty(), "an unsampled remote parent must not be exported" ); unsampled.shutdown(); } #[tokio::test(flavor = "current_thread")] async fn generates_a_sampled_root_span_for_absent_or_invalid_traceparent() { let absent = TracedRequest::new(get(probe)); let response = absent.send(&[]).await; let trace_id = TracedRequest::response_trace_id(&response); assert_request_span_is_exported(&absent, &trace_id); assert!( absent .exporter .named("http.request") .unwrap() .parent_span_id == SpanId::INVALID, "an absent traceparent must produce a root request span" ); absent.shutdown(); let invalid = TracedRequest::new(get(probe)); let response = invalid .send(&[("traceparent", "canary-invalid-traceparent")]) .await; let trace_id = TracedRequest::response_trace_id(&response); assert_ne!(trace_id, "canary-invalid-traceparent"); assert_request_span_is_exported(&invalid, &trace_id); invalid.shutdown(); } #[tokio::test(flavor = "current_thread")] async fn keeps_stage_child_spans_linked_to_the_inbound_request_span() { let request = TracedRequest::new(get(stage_probe)); let response = request.send(&[]).await; assert_eq!(response.status(), 200); let trace_id = TracedRequest::response_trace_id(&response); assert_request_span_is_exported(&request, &trace_id); let request_span = request.exporter.named("http.request").unwrap(); let stage_span = request .exporter .named("auth.resolve") .expect("stage spans must stay recorded and exported"); assert_eq!( stage_span.span_context.trace_id().to_string(), request_span.span_context.trace_id().to_string(), "stage spans must stay in the request trace" ); assert_eq!( stage_span.parent_span_id.to_string(), request_span.span_context.span_id().to_string(), "stage spans must stay children of the inbound request span" ); request.shutdown(); } #[tokio::test(flavor = "current_thread")] async fn keeps_stage_child_spans_linked_when_the_ambient_otel_context_is_suppressed() { let request = TracedRequest::new(get(stage_probe)); let _suppressed = Context::enter_telemetry_suppressed_scope(); let response = request.send(&[]).await; assert_eq!(response.status(), 200); let trace_id = TracedRequest::response_trace_id(&response); assert_request_span_is_exported(&request, &trace_id); let request_span = request.exporter.named("http.request").unwrap(); let stage_span = request.exporter.named("auth.resolve").unwrap_or_else(|| { panic!( "stage spans must stay recorded even with a suppressed ambient context; exported: {:?}", request .exporter .finished() .iter() .map(|span| span.name.to_string()) .collect::>() ) }); assert_eq!( stage_span.span_context.trace_id().to_string(), request_span.span_context.trace_id().to_string() ); assert_eq!( stage_span.parent_span_id.to_string(), request_span.span_context.span_id().to_string() ); request.shutdown(); }