Files
crank/crates/crank-observability/src/error_reporting.rs
T

645 lines
21 KiB
Rust

use std::{
borrow::Cow,
collections::BTreeMap,
fmt,
future::Future,
time::{Duration, SystemTime},
};
use sentry::{
ClientInitGuard, ClientOptions,
protocol::{Event, Level},
types::Dsn,
};
use thiserror::Error;
use crate::{RedactionLimits, ServiceIdentity, propagation::current_trace_id};
const CRITICAL_ERROR_MESSAGE: &str = "critical error";
const SENTRY_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(2);
// Sentry serializes SystemTime as a finite f64; this keeps conservative fixed headroom.
const MAX_SERIALIZED_TIMESTAMP_BYTES: usize = 32;
tokio::task_local! {
static REQUEST_ID: String;
static TRACE_ID: String;
}
pub struct SentryConfig {
dsn: Option<Dsn>,
}
impl SentryConfig {
pub fn parse(value: Option<&str>) -> Result<Self, SentryConfigError> {
let Some(value) = value.map(str::trim).filter(|value| !value.is_empty()) else {
return Ok(Self { dsn: None });
};
let dsn = value
.parse::<Dsn>()
.map_err(|_| SentryConfigError::InvalidDsn)?;
Ok(Self { dsn: Some(dsn) })
}
pub fn enabled(&self) -> bool {
self.dsn.is_some()
}
}
impl fmt::Debug for SentryConfig {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("SentryConfig")
.field("enabled", &self.enabled())
.finish()
}
}
#[derive(Debug, Error)]
pub enum SentryConfigError {
#[error("CRANK_SENTRY_DSN is not a valid Sentry DSN")]
InvalidDsn,
#[error("critical error event budget cannot hold the required fields")]
EventBudgetTooSmall,
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum CriticalErrorCategory {
Panic,
Startup,
Internal,
DataIntegrity,
}
impl CriticalErrorCategory {
const ALL: [Self; 4] = [
Self::Panic,
Self::Startup,
Self::Internal,
Self::DataIntegrity,
];
pub const fn as_str(self) -> &'static str {
match self {
Self::Panic => "panic",
Self::Startup => "startup",
Self::Internal => "internal",
Self::DataIntegrity => "data_integrity",
}
}
fn parse(value: &str) -> Option<Self> {
match value {
"panic" => Some(Self::Panic),
"startup" => Some(Self::Startup),
"internal" => Some(Self::Internal),
"data_integrity" => Some(Self::DataIntegrity),
_ => None,
}
}
}
pub fn capture_critical_error(category: CriticalErrorCategory) {
let mut tags = correlation_tags();
tags.insert("category".to_owned(), category.as_str().to_owned());
sentry::capture_event(Event {
level: Level::Error,
message: Some(CRITICAL_ERROR_MESSAGE.to_owned()),
fingerprint: Cow::Owned(vec![Cow::Borrowed(category.as_str())]),
tags,
..Event::default()
});
}
pub async fn with_request_correlation<F>(
request_id: String,
trace_id: String,
future: F,
) -> F::Output
where
F: Future,
{
if !valid_request_id(&request_id) || !valid_trace_id(&trace_id) {
return future.await;
}
REQUEST_ID
.scope(request_id, TRACE_ID.scope(trace_id, future))
.await
}
fn valid_request_id(value: &str) -> bool {
!value.is_empty()
&& value.len() <= 128
&& value
.bytes()
.all(|byte| (0x21..=0x7e).contains(&byte) && byte != b',' && byte != b';')
}
fn valid_trace_id(value: &str) -> bool {
value.len() == 32
&& value != "00000000000000000000000000000000"
&& value
.bytes()
.all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
}
pub fn current_request_correlation() -> (Option<String>, Option<String>) {
(
REQUEST_ID.try_with(Clone::clone).ok(),
TRACE_ID.try_with(Clone::clone).ok(),
)
}
pub(crate) fn init_sentry(
identity: &ServiceIdentity,
limits: RedactionLimits,
config: SentryConfig,
) -> Result<Option<ClientInitGuard>, SentryConfigError> {
let Some(dsn) = config.dsn else {
return Ok(None);
};
validate_critical_event_budget(identity, limits)?;
let identity = identity.clone();
let options = client_options(identity, limits);
Ok(Some(sentry::init((dsn, options))))
}
fn client_options(identity: ServiceIdentity, limits: RedactionLimits) -> ClientOptions {
let release = identity.version().to_owned();
let environment = identity.environment().to_owned();
let service = identity.service().to_owned();
let sanitizer_identity = identity.clone();
let mut options = ClientOptions::default();
options.release = Some(Cow::Owned(release));
options.environment = Some(Cow::Owned(environment));
options.server_name = Some(Cow::Owned(service));
options.traces_sampling_strategy = sentry::TracesSamplingStrategy::Disabled;
options.max_breadcrumbs = 0;
options.attach_stacktrace = false;
options.send_default_pii = false;
options.before_send = Some(std::sync::Arc::new(move |event| {
Some(sanitize_event(event, &sanitizer_identity, limits))
}));
options.shutdown_timeout = SENTRY_SHUTDOWN_TIMEOUT;
options.auto_session_tracking = false;
options.enable_logs = false;
options.enable_metrics = false;
options
}
fn sanitize_event(
event: Event<'static>,
identity: &ServiceIdentity,
limits: RedactionLimits,
) -> Event<'static> {
let category = event
.tags
.get("category")
.and_then(|value| CriticalErrorCategory::parse(value))
.unwrap_or_else(|| {
if event.exception.is_empty() {
CriticalErrorCategory::Internal
} else {
CriticalErrorCategory::Panic
}
});
let mut tags = correlation_tags().into_iter().collect::<BTreeMap<_, _>>();
tags.insert("service".to_owned(), identity.service().to_owned());
tags.insert("category".to_owned(), category.as_str().to_owned());
enforce_event_budget(
Event {
event_id: event.event_id,
level: Level::Error,
fingerprint: Cow::Owned(vec![
Cow::Owned(identity.service().to_owned()),
Cow::Borrowed(category.as_str()),
]),
message: Some(CRITICAL_ERROR_MESSAGE.to_owned()),
timestamp: event.timestamp,
server_name: Some(Cow::Owned(identity.service().to_owned())),
release: Some(Cow::Owned(identity.version().to_owned())),
environment: Some(Cow::Owned(identity.environment().to_owned())),
tags,
..Event::default()
},
limits.max_event_bytes,
)
}
fn correlation_tags() -> BTreeMap<String, String> {
let mut tags = BTreeMap::new();
if let Ok(request_id) = REQUEST_ID.try_with(Clone::clone) {
tags.insert("request_id".to_owned(), request_id);
}
if let Ok(trace_id) = TRACE_ID.try_with(Clone::clone) {
tags.insert("trace_id".to_owned(), trace_id);
} else if let Some(trace_id) = current_trace_id() {
tags.insert("trace_id".to_owned(), trace_id);
}
tags
}
fn enforce_event_budget(event: Event<'static>, max_event_bytes: usize) -> Event<'static> {
if serialized_event_len(&event) <= max_event_bytes {
return event;
}
// Startup validation reserves enough room for maximum canonical IDs. They
// are never evicted from a support event to satisfy a byte budget.
debug_assert!(serialized_event_len(&event) <= max_event_bytes);
event
}
fn validate_critical_event_budget(
identity: &ServiceIdentity,
limits: RedactionLimits,
) -> Result<(), SentryConfigError> {
if required_critical_event_budget(identity, limits) > limits.max_event_bytes {
return Err(SentryConfigError::EventBudgetTooSmall);
}
Ok(())
}
fn required_critical_event_budget(identity: &ServiceIdentity, limits: RedactionLimits) -> usize {
let unbounded_limits = RedactionLimits {
max_event_bytes: usize::MAX,
..limits
};
CriticalErrorCategory::ALL
.into_iter()
.map(|category| {
let mut event = sanitize_event(
Event {
tags: BTreeMap::from([("category".to_owned(), category.as_str().to_owned())]),
timestamp: SystemTime::UNIX_EPOCH,
..Event::default()
},
identity,
unbounded_limits,
);
event.tags.insert("request_id".to_owned(), "r".repeat(128));
event.tags.insert("trace_id".to_owned(), "a".repeat(32));
serialized_event_len(&event)
.saturating_add(MAX_SERIALIZED_TIMESTAMP_BYTES.saturating_sub(1))
})
.max()
.unwrap_or(usize::MAX)
}
fn serialized_event_len(event: &Event<'_>) -> usize {
serde_json::to_vec(event).map_or(usize::MAX, |serialized| serialized.len())
}
#[cfg(test)]
mod tests {
use std::{
collections::BTreeMap,
panic::AssertUnwindSafe,
sync::{
Arc,
atomic::{AtomicUsize, Ordering},
},
time::Duration,
};
use opentelemetry::trace::TracerProvider as _;
use opentelemetry_sdk::trace::SdkTracerProvider;
use sentry::{
Envelope, Hub,
protocol::{Breadcrumb, Context, Exception, Request, User, Value, Values},
};
use super::{
CriticalErrorCategory, capture_critical_error, client_options,
required_critical_event_budget, sanitize_event, validate_critical_event_budget,
with_request_correlation,
};
use crate::{
ObservabilityConfig, RedactionLimits, ServiceIdentity,
logging::build_subscriber_with_tracer,
};
fn identity() -> ServiceIdentity {
ServiceIdentity::try_new("admin-api", "1.2.3", "test").expect("valid identity")
}
#[test]
fn client_disables_non_error_telemetry() {
let options = client_options(identity(), RedactionLimits::default());
assert_eq!(options.max_breadcrumbs, 0);
assert!(!options.attach_stacktrace);
assert!(!options.send_default_pii);
assert!(!options.auto_session_tracking);
assert!(!options.enable_logs);
assert!(!options.enable_metrics);
assert!(options.before_send.is_some());
}
#[test]
fn sanitizer_uses_a_strict_allowlist() {
let mut tags = BTreeMap::new();
tags.insert("category".to_owned(), "data_integrity".to_owned());
tags.insert("secret".to_owned(), "must-not-leak".to_owned());
let mut contexts = BTreeMap::new();
contexts.insert(
"secret".to_owned(),
Context::Other(BTreeMap::from([(
"token".to_owned(),
Value::String("must-not-leak".to_owned()),
)])),
);
let event = sentry::protocol::Event {
message: Some("password=must-not-leak".to_owned()),
request: Some(Request::default()),
user: Some(User::default()),
breadcrumbs: Values {
values: vec![Breadcrumb::default()],
},
exception: Values {
values: vec![Exception {
value: Some("must-not-leak".to_owned()),
..Exception::default()
}],
},
contexts,
extra: BTreeMap::from([(
"payload".to_owned(),
Value::String("must-not-leak".to_owned()),
)]),
tags,
..sentry::protocol::Event::default()
};
let cleaned = sanitize_event(event, &identity(), RedactionLimits::default());
let serialized = serde_json::to_string(&cleaned).expect("serialize event");
assert_eq!(cleaned.message.as_deref(), Some("critical error"));
assert_eq!(
cleaned.tags.get("category").map(String::as_str),
Some(CriticalErrorCategory::DataIntegrity.as_str())
);
assert!(cleaned.request.is_none());
assert!(cleaned.user.is_none());
assert!(cleaned.breadcrumbs.is_empty());
assert!(cleaned.exception.is_empty());
assert!(cleaned.contexts.is_empty());
assert!(cleaned.extra.is_empty());
assert!(!serialized.contains("must-not-leak"));
assert!(!serialized.contains("password"));
}
#[test]
fn sanitizer_honours_total_event_budget() {
let limits = RedactionLimits {
max_string_bytes: 8 * 1024,
max_event_bytes: 512,
..RedactionLimits::default()
};
let event = sentry::protocol::Event {
tags: BTreeMap::from([
("category".to_owned(), "internal".to_owned()),
("request_id".to_owned(), "r".repeat(8 * 1024)),
("trace_id".to_owned(), "t".repeat(8 * 1024)),
]),
..sentry::protocol::Event::default()
};
let cleaned = sanitize_event(event, &identity(), limits);
let serialized = serde_json::to_vec(&cleaned).expect("serialize event");
assert!(serialized.len() <= limits.max_event_bytes);
assert_eq!(
cleaned.tags.get("category").map(String::as_str),
Some("internal")
);
assert_eq!(
cleaned.tags.get("service").map(String::as_str),
Some("admin-api")
);
assert!(!cleaned.tags.contains_key("request_id"));
assert!(!cleaned.tags.contains_key("trace_id"));
}
#[test]
fn expected_application_errors_do_not_create_critical_events() {
let options =
sentry::apply_defaults(client_options(identity(), RedactionLimits::default()));
let events = sentry::test::with_captured_events_options(
|| tracing::error!("ordinary product error"),
options,
);
assert!(events.is_empty());
}
#[test]
fn explicit_critical_error_is_correlated_and_sanitized() {
let options =
sentry::apply_defaults(client_options(identity(), RedactionLimits::default()));
let runtime = tokio::runtime::Builder::new_current_thread()
.build()
.expect("runtime");
let provider = SdkTracerProvider::builder().build();
let tracer = provider.tracer("critical-error-test");
let subscriber = build_subscriber_with_tracer(
ObservabilityConfig::new(identity(), "info", RedactionLimits::default()),
std::io::sink,
Some(tracer),
)
.expect("subscriber");
let dispatch = tracing::Dispatch::new(subscriber);
let events = sentry::test::with_captured_events_options(
|| {
tracing::dispatcher::with_default(&dispatch, || {
runtime.block_on(with_request_correlation(
"request-123".to_owned(),
"0af7651916cd43dd8448eb211c80319c".to_owned(),
async {
let span = tracing::info_span!(target: "crank::trace", "http.request");
let _span_guard = span.enter();
capture_critical_error(CriticalErrorCategory::DataIntegrity);
},
));
});
},
options,
);
assert_eq!(events.len(), 1);
let event = &events[0];
assert_eq!(event.message.as_deref(), Some("critical error"));
assert_eq!(
event.tags.get("category").map(String::as_str),
Some("data_integrity")
);
assert_eq!(
event
.fingerprint
.iter()
.map(AsRef::as_ref)
.collect::<Vec<&str>>(),
vec!["admin-api", "data_integrity"]
);
assert_eq!(
event.tags.get("request_id").map(String::as_str),
Some("request-123")
);
assert_eq!(event.tags.get("trace_id").map(String::len), Some(32));
assert_eq!(event.release.as_deref(), Some("1.2.3"));
assert_eq!(event.environment.as_deref(), Some("test"));
assert_eq!(event.server_name.as_deref(), Some("admin-api"));
}
#[test]
fn panic_creates_exactly_one_sanitized_critical_event() {
let options =
sentry::apply_defaults(client_options(identity(), RedactionLimits::default()));
let runtime = tokio::runtime::Builder::new_current_thread()
.build()
.expect("runtime");
let provider = SdkTracerProvider::builder().build();
let tracer = provider.tracer("panic-correlation-test");
let subscriber = build_subscriber_with_tracer(
ObservabilityConfig::new(identity(), "info", RedactionLimits::default()),
std::io::sink,
Some(tracer),
)
.expect("subscriber");
let dispatch = tracing::Dispatch::new(subscriber);
let events = sentry::test::with_captured_events_options(
|| {
let result = std::panic::catch_unwind(AssertUnwindSafe(|| {
tracing::dispatcher::with_default(&dispatch, || {
runtime.block_on(with_request_correlation(
"panic-request-123".to_owned(),
"0af7651916cd43dd8448eb211c80319c".to_owned(),
async {
let span =
tracing::info_span!(target: "crank::trace", "http.request");
let _span_guard = span.enter();
panic!("password=must-not-leak");
},
));
});
}));
assert!(result.is_err());
},
options,
);
assert_eq!(events.len(), 1);
let event = &events[0];
assert_eq!(
event.tags.get("category").map(String::as_str),
Some("panic")
);
assert_eq!(
event.tags.get("request_id").map(String::as_str),
Some("panic-request-123")
);
assert_eq!(event.tags.get("trace_id").map(String::len), Some(32));
let serialized = serde_json::to_string(event).expect("serialize event");
assert!(!serialized.contains("must-not-leak"));
assert!(!serialized.contains("password"));
provider.shutdown().expect("provider shutdown");
}
#[test]
fn receiver_failure_does_not_change_product_result_or_recurse() {
struct UnavailableTransport {
attempts: AtomicUsize,
}
impl sentry::Transport for UnavailableTransport {
fn send_envelope(&self, _envelope: Envelope) {
self.attempts.fetch_add(1, Ordering::Relaxed);
}
fn flush(&self, _timeout: Duration) -> bool {
false
}
}
let transport = Arc::new(UnavailableTransport {
attempts: AtomicUsize::new(0),
});
let mut options =
sentry::apply_defaults(client_options(identity(), RedactionLimits::default()));
options.dsn = Some(
"https://public@example.invalid/1"
.parse()
.expect("valid test DSN"),
);
options.transport = Some(Arc::new(transport.clone()));
let client = Arc::new(sentry::Client::from(options));
let hub = Arc::new(Hub::new(
Some(Arc::clone(&client)),
Arc::new(Default::default()),
));
let product_result = Hub::run(hub, || {
capture_critical_error(CriticalErrorCategory::Internal);
42
});
assert_eq!(product_result, 42);
assert_eq!(transport.attempts.load(Ordering::Relaxed), 1);
assert!(!client.close(Some(Duration::from_millis(10))));
assert_eq!(transport.attempts.load(Ordering::Relaxed), 1);
}
#[test]
fn critical_event_budget_rejects_identity_that_does_not_fit() {
let maximum_label = "a".repeat(64);
let identity =
ServiceIdentity::try_new(maximum_label.clone(), maximum_label.clone(), maximum_label)
.expect("maximum identity");
let limits = RedactionLimits {
max_event_bytes: 512,
..RedactionLimits::default()
};
assert!(matches!(
validate_critical_event_budget(&identity, limits),
Err(super::SentryConfigError::EventBudgetTooSmall)
));
validate_critical_event_budget(
&identity,
RedactionLimits {
max_event_bytes: 1024,
..limits
},
)
.expect("larger budget must hold the required identity");
}
#[test]
fn critical_event_budget_covers_every_category() {
let identity = identity();
let limits = RedactionLimits::default();
let required_budget = required_critical_event_budget(&identity, limits);
validate_critical_event_budget(
&identity,
RedactionLimits {
max_event_bytes: required_budget,
..limits
},
)
.expect("exact required budget must be accepted");
assert!(matches!(
validate_critical_event_budget(
&identity,
RedactionLimits {
max_event_bytes: required_budget - 1,
..limits
},
),
Err(super::SentryConfigError::EventBudgetTooSmall)
));
}
}