use super::*; use std::{collections::BTreeSet, time::Duration}; use axum::{ Router, body::Bytes, extract::Request, http::StatusCode, middleware::{self, Next}, routing::post, }; use crank_core::{ProductEventKind, WorkspaceId}; use crank_observability::{ OtlpBatchConfig, OtlpTraceConfig, ServiceIdentity, build_tracer_provider, }; use opentelemetry::trace::TracerProvider as _; use opentelemetry_proto::tonic::{ collector::trace::v1::ExportTraceServiceRequest, common::v1::{KeyValue, any_value}, }; use opentelemetry_sdk::trace::SdkTracerProvider; use prost::Message; use tokio::{net::TcpListener, sync::mpsc}; use tracing::instrument::WithSubscriber; use tracing_subscriber::layer::SubscriberExt; pub(super) const RAW_DOCUMENT_CANARY: &str = "openapi: 3.1.0 raw-document-canary"; pub(super) const USER_ID_CANARY: &str = "user-id-canary-should-not-export"; const REQUEST_ID: &str = "req_onboarding_optin_0001"; #[tokio::test(flavor = "multi_thread")] #[serial] async fn onboarding_http_event_keeps_default_trace_export_disabled() { let registry = test_registry().await; let workspace_id = WorkspaceId::new("ws_default"); let config = OtlpTraceConfig::default(); let identity = service_identity(); assert!(!config.is_enabled()); assert!( build_tracer_provider(&identity, &config) .expect("disabled OTLP config is valid") .is_none(), "default config must not construct an exporter or batch queue" ); let (collector_listener, _) = loopback_listener().await; let (dispatch, provider) = local_trace_dispatch(); let app = app_with_request_dispatch( build_test_app( registry.clone(), test_storage_root("onboarding_telemetry_default"), ), dispatch, ); let base_url = spawn_admin_api(app).await; let client = authorized_client(&base_url).await; let revision = onboarding_revision(&client, &base_url).await; reject_canary_payload(&client, &base_url, revision).await; let event = post_started_event(&client, &base_url, revision, false).await; assert_eq!(event.status(), reqwest::StatusCode::OK); assert_eq!(event.json::().await.unwrap()["accepted"], true); assert_event_is_local(®istry, &workspace_id, "ui:telemetry-default-event").await; assert!( tokio::time::timeout(Duration::from_millis(300), collector_listener.accept()) .await .is_err(), "default telemetry config must make no connection to the external collector" ); provider .shutdown() .expect("local trace provider shuts down"); } #[tokio::test(flavor = "multi_thread")] #[serial] async fn onboarding_http_event_exports_only_sanitized_trace_when_operator_opts_in() { let registry = test_registry().await; let workspace_id = WorkspaceId::new("ws_default"); let (endpoint, mut exported_requests, collector_task) = spawn_otlp_collector().await; let config = OtlpTraceConfig::try_new( Some(endpoint), Some("http/protobuf".to_owned()), Duration::from_secs(2), OtlpBatchConfig::try_new(128, 128, Duration::from_secs(60), Duration::from_secs(2)) .unwrap(), ) .unwrap(); let identity = service_identity(); assert!(config.is_enabled()); let (provider, tracer) = build_tracer_provider(&identity, &config) .expect("explicit OTLP config is valid") .expect("operator endpoint enables the real exporter"); let subscriber = tracing_subscriber::registry().with(tracing_opentelemetry::layer().with_tracer(tracer)); let dispatch = tracing::Dispatch::new(subscriber); let app = app_with_request_dispatch( build_test_app( registry.clone(), test_storage_root("onboarding_telemetry_optin"), ), dispatch, ); let base_url = spawn_admin_api(app).await; let client = authorized_client(&base_url).await; let revision = onboarding_revision(&client, &base_url).await; reject_canary_payload(&client, &base_url, revision).await; let event = post_started_event(&client, &base_url, revision, true).await; assert_eq!(event.status(), reqwest::StatusCode::OK); assert_eq!(event.json::().await.unwrap()["accepted"], true); assert_event_is_local(®istry, &workspace_id, "ui:telemetry-optin-event").await; provider .force_flush() .expect("flush trace spans to the operator collector"); provider.shutdown().expect("exporter shuts down cleanly"); let mut request_count = 0; let mut onboarding_request_id_exported = false; while let Ok(body) = exported_requests.try_recv() { request_count += 1; let export = ExportTraceServiceRequest::decode(body.as_ref()) .expect("collector receives valid OTLP protobuf"); for resource_spans in export.resource_spans { for scope_spans in resource_spans.scope_spans { for span in scope_spans.spans { assert_eq!(span.name, "http.request"); assert!(span.events.is_empty()); assert!(span.links.is_empty()); let attribute_keys: BTreeSet<&str> = span .attributes .iter() .map(|attribute| attribute.key.as_str()) .collect(); assert!( attribute_keys.iter().all(|key| *key == "request_id"), "request spans may export only the request_id attribute" ); if string_attribute(&span.attributes, "request_id") == Some(REQUEST_ID) { onboarding_request_id_exported = true; } } } } assert!(!body_contains(&body, RAW_DOCUMENT_CANARY)); assert!(!body_contains(&body, USER_ID_CANARY)); } assert!( request_count > 0, "explicit opt-in must reach the collector" ); assert!( onboarding_request_id_exported, "the collected trace must include the real onboarding HTTP request" ); collector_task.abort(); } fn service_identity() -> ServiceIdentity { ServiceIdentity::try_new("admin-api", "integration-test", "test") .expect("static service identity is valid") } fn local_trace_dispatch() -> (tracing::Dispatch, SdkTracerProvider) { let provider = SdkTracerProvider::builder().build(); let tracer = provider.tracer("crank"); let subscriber = tracing_subscriber::registry().with(tracing_opentelemetry::layer().with_tracer(tracer)); (tracing::Dispatch::new(subscriber), provider) } fn app_with_request_dispatch(app: Router, dispatch: tracing::Dispatch) -> Router { app.layer(middleware::from_fn(move |request: Request, next: Next| { let dispatch = dispatch.clone(); async move { next.run(request).with_subscriber(dispatch).await } })) } pub(super) async fn loopback_listener() -> (TcpListener, String) { let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); let endpoint = format!("http://{}/v1/traces", listener.local_addr().unwrap()); (listener, endpoint) } pub(super) async fn spawn_otlp_collector() -> ( String, mpsc::UnboundedReceiver, tokio::task::JoinHandle<()>, ) { let (listener, endpoint) = loopback_listener().await; let (sender, receiver) = mpsc::unbounded_channel(); let app = Router::new().route( "/v1/traces", post(move |body: Bytes| { let sender = sender.clone(); async move { let _ = sender.send(body); StatusCode::OK } }), ); let handle = tokio::spawn(async move { axum::serve(listener, app).await.unwrap(); }); (endpoint, receiver, handle) } pub(super) async fn onboarding_revision( client: &reqwest::Client, base_url: &(impl AsRef + ?Sized), ) -> i64 { let base_url = base_url.as_ref(); let response = client .get(format!("{base_url}/onboarding")) .send() .await .unwrap(); assert_eq!(response.status(), reqwest::StatusCode::OK); response.json::().await.unwrap()["revision"] .as_i64() .expect("onboarding returns a server revision") } pub(super) async fn reject_canary_payload( client: &reqwest::Client, base_url: &(impl AsRef + ?Sized), revision: i64, ) { let base_url = base_url.as_ref(); let response = client .post(format!("{base_url}/onboarding/events")) .json(&json!({ "event": "started", "idempotency_key": "ui:telemetry-canary-rejection", "expected_revision": revision, "user_id": USER_ID_CANARY, "openapi_document": RAW_DOCUMENT_CANARY, })) .send() .await .unwrap(); assert_eq!(response.status(), reqwest::StatusCode::UNPROCESSABLE_ENTITY); let body = response.text().await.unwrap(); assert!(!body.contains(USER_ID_CANARY)); assert!(!body.contains(RAW_DOCUMENT_CANARY)); } async fn post_started_event( client: &reqwest::Client, base_url: &(impl AsRef + ?Sized), revision: i64, with_request_id: bool, ) -> reqwest::Response { let base_url = base_url.as_ref(); let mut request = client .post(format!("{base_url}/onboarding/events")) .json(&json!({ "event": "started", "idempotency_key": if with_request_id { "ui:telemetry-optin-event" } else { "ui:telemetry-default-event" }, "expected_revision": revision, })); if with_request_id { request = request.header("x-request-id", REQUEST_ID); } request.send().await.unwrap() } async fn assert_event_is_local( registry: &crank_registry::PostgresRegistry, workspace_id: &WorkspaceId, idempotency_key: &str, ) { let record = registry .get_product_event_by_idempotency_key(workspace_id, idempotency_key) .await .unwrap() .expect("successful HTTP event is persisted in the local product event store"); assert_eq!(record.event.kind, ProductEventKind::OnboardingStarted); assert_eq!(record.event.idempotency_key, idempotency_key); } pub(super) fn string_attribute<'a>(attributes: &'a [KeyValue], key: &str) -> Option<&'a str> { attributes.iter().find_map(|attribute| { (attribute.key == key) .then_some(attribute.value.as_ref()?.value.as_ref()?) .and_then(|value| match value { any_value::Value::StringValue(value) => Some(value.as_str()), _ => None, }) }) } pub(super) fn body_contains(body: &[u8], canary: &str) -> bool { body.windows(canary.len()) .any(|window| window == canary.as_bytes()) }