use super::telemetry::{ RAW_DOCUMENT_CANARY, USER_ID_CANARY, body_contains, loopback_listener, onboarding_revision, reject_canary_payload, string_attribute, }; use super::*; use axum::{ Router, body::{self, Bytes}, extract::Request, http::StatusCode, routing::any, }; use opentelemetry_proto::tonic::collector::trace::v1::ExportTraceServiceRequest; use prost::Message; use std::{ collections::BTreeSet, fs::{self, OpenOptions}, net::TcpListener as StdTcpListener, os::unix::fs::{OpenOptionsExt, PermissionsExt}, path::PathBuf, process::{Child, Command, Stdio}, time::{Duration, Instant}, }; const MASTER_KEY: &str = "test-master-key-00000000000000000000000000000000"; const SESSION_SECRET: &str = "test-session-secret"; const PASSWORD_PEPPER: &str = "test-password-pepper"; const ADMIN_EMAIL: &str = "owner@crank.local"; const ADMIN_PASSWORD: &str = "test-password"; const REQUEST_ID: &str = "req_admin_process_optin_0001"; struct AdminProcess { child: Child, log_path: PathBuf, secrets: Vec, } impl AdminProcess { fn stop_and_assert_logs_are_safe(&mut self) { let _ = self.child.kill(); self.child.wait().expect("admin-api child must be reaped"); let logs = fs::read_to_string(&self.log_path).expect("child log must be readable"); for secret in &self.secrets { assert!( !logs.contains(secret), "admin-api process log must not contain configured secrets or request canaries" ); } } } impl Drop for AdminProcess { fn drop(&mut self) { let _ = self.child.kill(); let _ = self.child.wait(); } } #[tokio::test(flavor = "multi_thread")] #[serial] async fn production_admin_process_keeps_default_otlp_export_disabled() { let (registry, database_url) = test_registry_with_single_connection().await; let _registry = registry; let storage_root = test_storage_root("onboarding_telemetry_startup_default"); let (collector_listener, _) = loopback_listener().await; let (mut process, base_url) = start_admin_process(&database_url, &storage_root, None); wait_until_ready(&mut process, &base_url).await; let workspace_url = format!("{base_url}/api/admin/workspaces/ws_default"); let client = authorized_client(&workspace_url).await; let revision = onboarding_revision(&client, &workspace_url).await; reject_canary_payload(&client, &workspace_url, revision).await; let response = post_startup_event(&client, &workspace_url, revision, false).await; assert_eq!(response.status(), reqwest::StatusCode::OK); assert_eq!(response.json::().await.unwrap()["accepted"], true); assert!( tokio::time::timeout(Duration::from_secs(2), collector_listener.accept()) .await .is_err(), "default production startup must not connect to an OTLP collector" ); process.stop_and_assert_logs_are_safe(); } #[tokio::test(flavor = "multi_thread")] #[serial] async fn production_admin_process_exports_only_sanitized_request_trace_when_opted_in() { let (registry, database_url) = test_registry_with_single_connection().await; let _registry = registry; let storage_root = test_storage_root("onboarding_telemetry_startup_optin"); let (endpoint, mut exported_requests, collector_task) = spawn_startup_collector().await; let (mut process, base_url) = start_admin_process(&database_url, &storage_root, Some(&endpoint)); wait_until_ready(&mut process, &base_url).await; let workspace_url = format!("{base_url}/api/admin/workspaces/ws_default"); let client = authorized_client(&workspace_url).await; let revision = onboarding_revision(&client, &workspace_url).await; reject_canary_payload(&client, &workspace_url, revision).await; let response = post_startup_event(&client, &workspace_url, revision, true).await; assert_eq!(response.status(), reqwest::StatusCode::OK); assert_eq!(response.json::().await.unwrap()["accepted"], true); let mut request_id_exported = false; tokio::time::timeout(Duration::from_secs(12), async { while !request_id_exported { let (path, body) = exported_requests .recv() .await .expect("collector task must remain active"); assert_eq!(path, "/v1/traces", "trace-specific endpoint path is exact"); let export = ExportTraceServiceRequest::decode(body.as_ref()) .expect("production process must send valid OTLP protobuf"); for resource_spans in export.resource_spans { for scope_spans in resource_spans.scope_spans { for span in scope_spans.spans { assert_eq!(span.name, "http.request"); assert!(span.events.is_empty()); assert!(span.links.is_empty()); let attribute_keys: BTreeSet<&str> = span .attributes .iter() .map(|attribute| attribute.key.as_str()) .collect(); assert!( attribute_keys.iter().all(|key| *key == "request_id"), "production request spans may export only the request_id attribute" ); if string_attribute(&span.attributes, "request_id") == Some(REQUEST_ID) { request_id_exported = true; } let encoded = format!("{} {:?}", span.name, span.attributes); assert!(!encoded.contains(USER_ID_CANARY)); assert!(!encoded.contains(RAW_DOCUMENT_CANARY)); } } } assert!(!body_contains(&body, USER_ID_CANARY)); assert!(!body_contains(&body, RAW_DOCUMENT_CANARY)); } }) .await .expect("production exporter must send the onboarding request in a bounded OTLP batch"); assert!( request_id_exported, "collector must receive the trace for the onboarding event request" ); process.stop_and_assert_logs_are_safe(); collector_task.abort(); } fn start_admin_process( database_url: &str, storage_root: &std::path::Path, otlp_endpoint: Option<&str>, ) -> (AdminProcess, String) { fs::create_dir_all(storage_root).expect("isolated storage root must be created"); #[cfg(unix)] fs::set_permissions(storage_root, fs::Permissions::from_mode(0o700)) .expect("isolated storage root must be private"); let listener = StdTcpListener::bind("127.0.0.1:0").expect("reserve an ephemeral API port"); let address = listener.local_addr().unwrap(); drop(listener); let base_url = format!("http://{address}"); let log_path = storage_root.join("admin-api.log"); let stdout = OpenOptions::new() .write(true) .create_new(true) .mode(0o600) .open(&log_path) .expect("private child log must be created"); let stderr = stdout.try_clone().expect("child stderr log handle"); let mut command = Command::new(env!("CARGO_BIN_EXE_admin-api")); command .env_clear() .env("CRANK_DATABASE_URL", database_url) .env("CRANK_ADMIN_BIND", address.to_string()) .env("CRANK_STORAGE_ROOT", storage_root) .env("CRANK_MASTER_KEY", MASTER_KEY) .env("CRANK_SESSION_SECRET", SESSION_SECRET) .env("CRANK_PASSWORD_PEPPER", PASSWORD_PEPPER) .env("CRANK_SESSION_TTL_HOURS", "24") .env("CRANK_BOOTSTRAP_ADMIN_EMAIL", ADMIN_EMAIL) .env("CRANK_BOOTSTRAP_ADMIN_PASSWORD", ADMIN_PASSWORD) .env("CRANK_BOOTSTRAP_ADMIN_DISPLAY_NAME", "Integration Test") .env("CRANK_METRICS_ENABLED", "false") .env("CRANK_BASE_URL", &base_url) .env("CRANK_LOG_LEVEL", "info") .stdout(Stdio::from(stdout)) .stderr(Stdio::from(stderr)); if let Some(endpoint) = otlp_endpoint { command .env("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", endpoint) .env("OTEL_EXPORTER_OTLP_TRACES_PROTOCOL", "http/protobuf") .env("OTEL_BSP_MAX_QUEUE_SIZE", "16") .env("OTEL_BSP_MAX_EXPORT_BATCH_SIZE", "1") .env("OTEL_BSP_SCHEDULE_DELAY", "100") .env("OTEL_BSP_EXPORT_TIMEOUT", "2000"); } let child = command.spawn().expect("production admin-api binary starts"); ( AdminProcess { child, log_path, secrets: vec![ database_url.to_owned(), MASTER_KEY.to_owned(), SESSION_SECRET.to_owned(), PASSWORD_PEPPER.to_owned(), ADMIN_PASSWORD.to_owned(), USER_ID_CANARY.to_owned(), RAW_DOCUMENT_CANARY.to_owned(), ], }, base_url, ) } async fn spawn_startup_collector() -> ( String, tokio::sync::mpsc::UnboundedReceiver<(String, Bytes)>, tokio::task::JoinHandle<()>, ) { let (listener, endpoint) = loopback_listener().await; let (sender, receiver) = tokio::sync::mpsc::unbounded_channel(); let app = Router::new().fallback(any(move |request: Request| { let sender = sender.clone(); async move { let path = request.uri().path().to_owned(); let body = body::to_bytes(request.into_body(), usize::MAX) .await .expect("collector request body must be readable"); let _ = sender.send((path, body)); StatusCode::OK } })); let handle = tokio::spawn(async move { axum::serve(listener, app).await.unwrap(); }); (endpoint, receiver, handle) } async fn wait_until_ready(process: &mut AdminProcess, base_url: &str) { let deadline = Instant::now() + Duration::from_secs(45); loop { if let Some(status) = process.child.try_wait().expect("child status is readable") { panic!("production admin-api exited before readiness: {status}"); } if reqwest::Client::new() .get(format!("{base_url}/health")) .timeout(Duration::from_millis(500)) .send() .await .is_ok_and(|response| response.status().is_success()) { return; } assert!( Instant::now() < deadline, "production admin-api readiness deadline" ); tokio::time::sleep(Duration::from_millis(100)).await; } } async fn post_startup_event( client: &reqwest::Client, workspace_url: &str, revision: i64, with_request_id: bool, ) -> reqwest::Response { let mut request = client .post(format!("{workspace_url}/onboarding/events")) .json(&json!({ "event": "started", "idempotency_key": if with_request_id { "ui:telemetry-process-optin" } else { "ui:telemetry-process-default" }, "expected_revision": revision, })); if with_request_id { request = request.header("x-request-id", REQUEST_ID); } request.send().await.unwrap() }