use std::{ borrow::Cow, collections::BTreeMap, fmt, future::Future, time::{Duration, SystemTime}, }; use sentry::{ ClientInitGuard, ClientOptions, protocol::{Event, Level}, types::Dsn, }; use thiserror::Error; use crate::{RedactionLimits, ServiceIdentity, propagation::current_trace_id}; const CRITICAL_ERROR_MESSAGE: &str = "critical error"; const SENTRY_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(2); // Sentry serializes SystemTime as a finite f64; this keeps conservative fixed headroom. const MAX_SERIALIZED_TIMESTAMP_BYTES: usize = 32; tokio::task_local! { static REQUEST_ID: String; static TRACE_ID: String; } pub struct SentryConfig { dsn: Option, } impl SentryConfig { pub fn parse(value: Option<&str>) -> Result { let Some(value) = value.map(str::trim).filter(|value| !value.is_empty()) else { return Ok(Self { dsn: None }); }; let dsn = value .parse::() .map_err(|_| SentryConfigError::InvalidDsn)?; Ok(Self { dsn: Some(dsn) }) } pub fn enabled(&self) -> bool { self.dsn.is_some() } } impl fmt::Debug for SentryConfig { fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { formatter .debug_struct("SentryConfig") .field("enabled", &self.enabled()) .finish() } } #[derive(Debug, Error)] pub enum SentryConfigError { #[error("CRANK_SENTRY_DSN is not a valid Sentry DSN")] InvalidDsn, #[error("critical error event budget cannot hold the required fields")] EventBudgetTooSmall, } #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum CriticalErrorCategory { Panic, Startup, Internal, DataIntegrity, } impl CriticalErrorCategory { const ALL: [Self; 4] = [ Self::Panic, Self::Startup, Self::Internal, Self::DataIntegrity, ]; pub const fn as_str(self) -> &'static str { match self { Self::Panic => "panic", Self::Startup => "startup", Self::Internal => "internal", Self::DataIntegrity => "data_integrity", } } fn parse(value: &str) -> Option { match value { "panic" => Some(Self::Panic), "startup" => Some(Self::Startup), "internal" => Some(Self::Internal), "data_integrity" => Some(Self::DataIntegrity), _ => None, } } } pub fn capture_critical_error(category: CriticalErrorCategory) { let mut tags = correlation_tags(); tags.insert("category".to_owned(), category.as_str().to_owned()); sentry::capture_event(Event { level: Level::Error, message: Some(CRITICAL_ERROR_MESSAGE.to_owned()), fingerprint: Cow::Owned(vec![Cow::Borrowed(category.as_str())]), tags, ..Event::default() }); } pub async fn with_request_correlation( request_id: String, trace_id: String, future: F, ) -> F::Output where F: Future, { if !valid_request_id(&request_id) || !valid_trace_id(&trace_id) { return future.await; } REQUEST_ID .scope(request_id, TRACE_ID.scope(trace_id, future)) .await } fn valid_request_id(value: &str) -> bool { !value.is_empty() && value.len() <= 128 && value .bytes() .all(|byte| (0x21..=0x7e).contains(&byte) && byte != b',' && byte != b';') } fn valid_trace_id(value: &str) -> bool { value.len() == 32 && value != "00000000000000000000000000000000" && value .bytes() .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) } pub fn current_request_correlation() -> (Option, Option) { ( REQUEST_ID.try_with(Clone::clone).ok(), TRACE_ID.try_with(Clone::clone).ok(), ) } pub(crate) fn init_sentry( identity: &ServiceIdentity, limits: RedactionLimits, config: SentryConfig, ) -> Result, SentryConfigError> { let Some(dsn) = config.dsn else { return Ok(None); }; validate_critical_event_budget(identity, limits)?; let identity = identity.clone(); let options = client_options(identity, limits); Ok(Some(sentry::init((dsn, options)))) } fn client_options(identity: ServiceIdentity, limits: RedactionLimits) -> ClientOptions { let release = identity.version().to_owned(); let environment = identity.environment().to_owned(); let service = identity.service().to_owned(); let sanitizer_identity = identity.clone(); let mut options = ClientOptions::default(); options.release = Some(Cow::Owned(release)); options.environment = Some(Cow::Owned(environment)); options.server_name = Some(Cow::Owned(service)); options.traces_sampling_strategy = sentry::TracesSamplingStrategy::Disabled; options.max_breadcrumbs = 0; options.attach_stacktrace = false; options.send_default_pii = false; options.before_send = Some(std::sync::Arc::new(move |event| { Some(sanitize_event(event, &sanitizer_identity, limits)) })); options.shutdown_timeout = SENTRY_SHUTDOWN_TIMEOUT; options.auto_session_tracking = false; options.enable_logs = false; options.enable_metrics = false; options } fn sanitize_event( event: Event<'static>, identity: &ServiceIdentity, limits: RedactionLimits, ) -> Event<'static> { let category = event .tags .get("category") .and_then(|value| CriticalErrorCategory::parse(value)) .unwrap_or_else(|| { if event.exception.is_empty() { CriticalErrorCategory::Internal } else { CriticalErrorCategory::Panic } }); let mut tags = correlation_tags().into_iter().collect::>(); tags.insert("service".to_owned(), identity.service().to_owned()); tags.insert("category".to_owned(), category.as_str().to_owned()); enforce_event_budget( Event { event_id: event.event_id, level: Level::Error, fingerprint: Cow::Owned(vec![ Cow::Owned(identity.service().to_owned()), Cow::Borrowed(category.as_str()), ]), message: Some(CRITICAL_ERROR_MESSAGE.to_owned()), timestamp: event.timestamp, server_name: Some(Cow::Owned(identity.service().to_owned())), release: Some(Cow::Owned(identity.version().to_owned())), environment: Some(Cow::Owned(identity.environment().to_owned())), tags, ..Event::default() }, limits.max_event_bytes, ) } fn correlation_tags() -> BTreeMap { let mut tags = BTreeMap::new(); if let Ok(request_id) = REQUEST_ID.try_with(Clone::clone) { tags.insert("request_id".to_owned(), request_id); } if let Ok(trace_id) = TRACE_ID.try_with(Clone::clone) { tags.insert("trace_id".to_owned(), trace_id); } else if let Some(trace_id) = current_trace_id() { tags.insert("trace_id".to_owned(), trace_id); } tags } fn enforce_event_budget(event: Event<'static>, max_event_bytes: usize) -> Event<'static> { if serialized_event_len(&event) <= max_event_bytes { return event; } // Startup validation reserves enough room for maximum canonical IDs. They // are never evicted from a support event to satisfy a byte budget. debug_assert!(serialized_event_len(&event) <= max_event_bytes); event } fn validate_critical_event_budget( identity: &ServiceIdentity, limits: RedactionLimits, ) -> Result<(), SentryConfigError> { if required_critical_event_budget(identity, limits) > limits.max_event_bytes { return Err(SentryConfigError::EventBudgetTooSmall); } Ok(()) } fn required_critical_event_budget(identity: &ServiceIdentity, limits: RedactionLimits) -> usize { let unbounded_limits = RedactionLimits { max_event_bytes: usize::MAX, ..limits }; CriticalErrorCategory::ALL .into_iter() .map(|category| { let mut event = sanitize_event( Event { tags: BTreeMap::from([("category".to_owned(), category.as_str().to_owned())]), timestamp: SystemTime::UNIX_EPOCH, ..Event::default() }, identity, unbounded_limits, ); event.tags.insert("request_id".to_owned(), "r".repeat(128)); event.tags.insert("trace_id".to_owned(), "a".repeat(32)); serialized_event_len(&event) .saturating_add(MAX_SERIALIZED_TIMESTAMP_BYTES.saturating_sub(1)) }) .max() .unwrap_or(usize::MAX) } fn serialized_event_len(event: &Event<'_>) -> usize { serde_json::to_vec(event).map_or(usize::MAX, |serialized| serialized.len()) } #[cfg(test)] mod tests { use std::{ collections::BTreeMap, panic::AssertUnwindSafe, sync::{ Arc, atomic::{AtomicUsize, Ordering}, }, time::Duration, }; use opentelemetry::trace::TracerProvider as _; use opentelemetry_sdk::trace::SdkTracerProvider; use sentry::{ Envelope, Hub, protocol::{Breadcrumb, Context, Exception, Request, User, Value, Values}, }; use super::{ CriticalErrorCategory, capture_critical_error, client_options, required_critical_event_budget, sanitize_event, validate_critical_event_budget, with_request_correlation, }; use crate::{ ObservabilityConfig, RedactionLimits, ServiceIdentity, logging::build_subscriber_with_tracer, }; fn identity() -> ServiceIdentity { ServiceIdentity::try_new("admin-api", "1.2.3", "test").expect("valid identity") } #[test] fn client_disables_non_error_telemetry() { let options = client_options(identity(), RedactionLimits::default()); assert_eq!(options.max_breadcrumbs, 0); assert!(!options.attach_stacktrace); assert!(!options.send_default_pii); assert!(!options.auto_session_tracking); assert!(!options.enable_logs); assert!(!options.enable_metrics); assert!(options.before_send.is_some()); } #[test] fn sanitizer_uses_a_strict_allowlist() { let mut tags = BTreeMap::new(); tags.insert("category".to_owned(), "data_integrity".to_owned()); tags.insert("secret".to_owned(), "must-not-leak".to_owned()); let mut contexts = BTreeMap::new(); contexts.insert( "secret".to_owned(), Context::Other(BTreeMap::from([( "token".to_owned(), Value::String("must-not-leak".to_owned()), )])), ); let event = sentry::protocol::Event { message: Some("password=must-not-leak".to_owned()), request: Some(Request::default()), user: Some(User::default()), breadcrumbs: Values { values: vec![Breadcrumb::default()], }, exception: Values { values: vec![Exception { value: Some("must-not-leak".to_owned()), ..Exception::default() }], }, contexts, extra: BTreeMap::from([( "payload".to_owned(), Value::String("must-not-leak".to_owned()), )]), tags, ..sentry::protocol::Event::default() }; let cleaned = sanitize_event(event, &identity(), RedactionLimits::default()); let serialized = serde_json::to_string(&cleaned).expect("serialize event"); assert_eq!(cleaned.message.as_deref(), Some("critical error")); assert_eq!( cleaned.tags.get("category").map(String::as_str), Some(CriticalErrorCategory::DataIntegrity.as_str()) ); assert!(cleaned.request.is_none()); assert!(cleaned.user.is_none()); assert!(cleaned.breadcrumbs.is_empty()); assert!(cleaned.exception.is_empty()); assert!(cleaned.contexts.is_empty()); assert!(cleaned.extra.is_empty()); assert!(!serialized.contains("must-not-leak")); assert!(!serialized.contains("password")); } #[test] fn sanitizer_honours_total_event_budget() { let limits = RedactionLimits { max_string_bytes: 8 * 1024, max_event_bytes: 512, ..RedactionLimits::default() }; let event = sentry::protocol::Event { tags: BTreeMap::from([ ("category".to_owned(), "internal".to_owned()), ("request_id".to_owned(), "r".repeat(8 * 1024)), ("trace_id".to_owned(), "t".repeat(8 * 1024)), ]), ..sentry::protocol::Event::default() }; let cleaned = sanitize_event(event, &identity(), limits); let serialized = serde_json::to_vec(&cleaned).expect("serialize event"); assert!(serialized.len() <= limits.max_event_bytes); assert_eq!( cleaned.tags.get("category").map(String::as_str), Some("internal") ); assert_eq!( cleaned.tags.get("service").map(String::as_str), Some("admin-api") ); assert!(!cleaned.tags.contains_key("request_id")); assert!(!cleaned.tags.contains_key("trace_id")); } #[test] fn expected_application_errors_do_not_create_critical_events() { let options = sentry::apply_defaults(client_options(identity(), RedactionLimits::default())); let events = sentry::test::with_captured_events_options( || tracing::error!("ordinary product error"), options, ); assert!(events.is_empty()); } #[test] fn explicit_critical_error_is_correlated_and_sanitized() { let options = sentry::apply_defaults(client_options(identity(), RedactionLimits::default())); let runtime = tokio::runtime::Builder::new_current_thread() .build() .expect("runtime"); let provider = SdkTracerProvider::builder().build(); let tracer = provider.tracer("critical-error-test"); let subscriber = build_subscriber_with_tracer( ObservabilityConfig::new(identity(), "info", RedactionLimits::default()), std::io::sink, Some(tracer), ) .expect("subscriber"); let dispatch = tracing::Dispatch::new(subscriber); let events = sentry::test::with_captured_events_options( || { tracing::dispatcher::with_default(&dispatch, || { runtime.block_on(with_request_correlation( "request-123".to_owned(), "0af7651916cd43dd8448eb211c80319c".to_owned(), async { let span = tracing::info_span!(target: "crank::trace", "http.request"); let _span_guard = span.enter(); capture_critical_error(CriticalErrorCategory::DataIntegrity); }, )); }); }, options, ); assert_eq!(events.len(), 1); let event = &events[0]; assert_eq!(event.message.as_deref(), Some("critical error")); assert_eq!( event.tags.get("category").map(String::as_str), Some("data_integrity") ); assert_eq!( event .fingerprint .iter() .map(AsRef::as_ref) .collect::>(), vec!["admin-api", "data_integrity"] ); assert_eq!( event.tags.get("request_id").map(String::as_str), Some("request-123") ); assert_eq!(event.tags.get("trace_id").map(String::len), Some(32)); assert_eq!(event.release.as_deref(), Some("1.2.3")); assert_eq!(event.environment.as_deref(), Some("test")); assert_eq!(event.server_name.as_deref(), Some("admin-api")); } #[test] fn panic_creates_exactly_one_sanitized_critical_event() { let options = sentry::apply_defaults(client_options(identity(), RedactionLimits::default())); let runtime = tokio::runtime::Builder::new_current_thread() .build() .expect("runtime"); let provider = SdkTracerProvider::builder().build(); let tracer = provider.tracer("panic-correlation-test"); let subscriber = build_subscriber_with_tracer( ObservabilityConfig::new(identity(), "info", RedactionLimits::default()), std::io::sink, Some(tracer), ) .expect("subscriber"); let dispatch = tracing::Dispatch::new(subscriber); let events = sentry::test::with_captured_events_options( || { let result = std::panic::catch_unwind(AssertUnwindSafe(|| { tracing::dispatcher::with_default(&dispatch, || { runtime.block_on(with_request_correlation( "panic-request-123".to_owned(), "0af7651916cd43dd8448eb211c80319c".to_owned(), async { let span = tracing::info_span!(target: "crank::trace", "http.request"); let _span_guard = span.enter(); panic!("password=must-not-leak"); }, )); }); })); assert!(result.is_err()); }, options, ); assert_eq!(events.len(), 1); let event = &events[0]; assert_eq!( event.tags.get("category").map(String::as_str), Some("panic") ); assert_eq!( event.tags.get("request_id").map(String::as_str), Some("panic-request-123") ); assert_eq!(event.tags.get("trace_id").map(String::len), Some(32)); let serialized = serde_json::to_string(event).expect("serialize event"); assert!(!serialized.contains("must-not-leak")); assert!(!serialized.contains("password")); provider.shutdown().expect("provider shutdown"); } #[test] fn receiver_failure_does_not_change_product_result_or_recurse() { struct UnavailableTransport { attempts: AtomicUsize, } impl sentry::Transport for UnavailableTransport { fn send_envelope(&self, _envelope: Envelope) { self.attempts.fetch_add(1, Ordering::Relaxed); } fn flush(&self, _timeout: Duration) -> bool { false } } let transport = Arc::new(UnavailableTransport { attempts: AtomicUsize::new(0), }); let mut options = sentry::apply_defaults(client_options(identity(), RedactionLimits::default())); options.dsn = Some( "https://public@example.invalid/1" .parse() .expect("valid test DSN"), ); options.transport = Some(Arc::new(transport.clone())); let client = Arc::new(sentry::Client::from(options)); let hub = Arc::new(Hub::new( Some(Arc::clone(&client)), Arc::new(Default::default()), )); let product_result = Hub::run(hub, || { capture_critical_error(CriticalErrorCategory::Internal); 42 }); assert_eq!(product_result, 42); assert_eq!(transport.attempts.load(Ordering::Relaxed), 1); assert!(!client.close(Some(Duration::from_millis(10)))); assert_eq!(transport.attempts.load(Ordering::Relaxed), 1); } #[test] fn critical_event_budget_rejects_identity_that_does_not_fit() { let maximum_label = "a".repeat(64); let identity = ServiceIdentity::try_new(maximum_label.clone(), maximum_label.clone(), maximum_label) .expect("maximum identity"); let limits = RedactionLimits { max_event_bytes: 512, ..RedactionLimits::default() }; assert!(matches!( validate_critical_event_budget(&identity, limits), Err(super::SentryConfigError::EventBudgetTooSmall) )); validate_critical_event_budget( &identity, RedactionLimits { max_event_bytes: 1024, ..limits }, ) .expect("larger budget must hold the required identity"); } #[test] fn critical_event_budget_covers_every_category() { let identity = identity(); let limits = RedactionLimits::default(); let required_budget = required_critical_event_budget(&identity, limits); validate_critical_event_budget( &identity, RedactionLimits { max_event_bytes: required_budget, ..limits }, ) .expect("exact required budget must be accepted"); assert!(matches!( validate_critical_event_budget( &identity, RedactionLimits { max_event_bytes: required_budget - 1, ..limits }, ), Err(super::SentryConfigError::EventBudgetTooSmall) )); } }