diff --git a/apps/admin-api/src/main.rs b/apps/admin-api/src/main.rs index 6dc9c42..3dfc097 100644 --- a/apps/admin-api/src/main.rs +++ b/apps/admin-api/src/main.rs @@ -21,6 +21,8 @@ use sqlx::postgres::PgConnectOptions; use tokio::net::TcpListener; use tracing::{info, warn}; +const MAX_INVOCATION_LOG_RETENTION_DAYS: i64 = 36_500; + #[tokio::main] async fn main() -> Result<(), Box> { let observability = crank_observability::init(ObservabilityConfig::from_env( @@ -106,8 +108,7 @@ async fn run( .with_outbound_http_policy(outbound_http_policy) .with_identity_provider(std::sync::Arc::new(identity_provider)) .build(); - let invocation_log_retention_days = - positive_i64_from_env("CRANK_INVOCATION_LOG_RETENTION_DAYS", 30)?; + let invocation_log_retention_days = invocation_log_retention_days_from_env()?; service.bootstrap_admin_user().await?; if env_flag("CRANK_DEMO_SEED") { service.seed_demo_assets().await?; @@ -159,17 +160,17 @@ async fn run( Ok(()) } -fn positive_i64_from_env( - name: &'static str, - default: i64, -) -> Result> { - let value = match env::var(name) { +fn invocation_log_retention_days_from_env() -> Result> { + const NAME: &str = "CRANK_INVOCATION_LOG_RETENTION_DAYS"; + let value = match env::var(NAME) { Ok(raw) => raw.parse::()?, - Err(env::VarError::NotPresent) => default, + Err(env::VarError::NotPresent) => 30, Err(error) => return Err(error.into()), }; - if value <= 0 { - return Err(format!("{name} must be greater than zero").into()); + if !(1..=MAX_INVOCATION_LOG_RETENTION_DAYS).contains(&value) { + return Err( + format!("{NAME} must be between 1 and {MAX_INVOCATION_LOG_RETENTION_DAYS}").into(), + ); } Ok(value) } diff --git a/apps/admin-api/src/request_context.rs b/apps/admin-api/src/request_context.rs index efa2818..d55e620 100644 --- a/apps/admin-api/src/request_context.rs +++ b/apps/admin-api/src/request_context.rs @@ -1,5 +1,5 @@ use axum::{ - extract::Request, + extract::{MatchedPath, Request}, http::{HeaderName, HeaderValue}, middleware::Next, response::Response, @@ -19,7 +19,11 @@ pub async fn apply_request_context(mut request: Request, next: Next) -> Response request_id: RequestId::resolve_from_headers(request.headers()).into_string(), }; let method = request.method().clone(); - let path = request.uri().path().to_owned(); + let route = request + .extensions() + .get::() + .map_or("unmatched", MatchedPath::as_str) + .to_owned(); let span = info_span!( target: "crank::trace", "http.request", @@ -34,7 +38,7 @@ pub async fn apply_request_context(mut request: Request, next: Next) -> Response name: "admin.request.completed", request_id = %context.request_id, method = %method, - path, + route, status = response.status().as_u16(), "admin request completed" ); diff --git a/apps/admin-api/tests/integration/request_context.rs b/apps/admin-api/tests/integration/request_context.rs index 08864fc..fa0fe44 100644 --- a/apps/admin-api/tests/integration/request_context.rs +++ b/apps/admin-api/tests/integration/request_context.rs @@ -65,6 +65,7 @@ async fn logs_request_completion_and_rejects_untrusted_values() { .unwrap(); assert_eq!(event["request_id"], "req_admin_trace_123"); assert_eq!(event["fields"]["status"], 200); + assert_eq!(event["fields"]["route"], "/probe"); let invalid_response = app .oneshot( diff --git a/crates/crank-observability/src/error_reporting.rs b/crates/crank-observability/src/error_reporting.rs index 5f6e810..fd18626 100644 --- a/crates/crank-observability/src/error_reporting.rs +++ b/crates/crank-observability/src/error_reporting.rs @@ -1,4 +1,10 @@ -use std::{borrow::Cow, collections::BTreeMap, env, fmt, future::Future, time::Duration}; +use std::{ + borrow::Cow, + collections::BTreeMap, + env, fmt, + future::Future, + time::{Duration, SystemTime}, +}; use sentry::{ ClientInitGuard, ClientOptions, @@ -14,6 +20,8 @@ use crate::{ const SENTRY_DSN_ENV: &str = "CRANK_SENTRY_DSN"; const CRITICAL_ERROR_MESSAGE: &str = "critical error"; const SENTRY_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(2); +// Sentry serializes SystemTime as a finite f64; this keeps conservative fixed headroom. +const MAX_SERIALIZED_TIMESTAMP_BYTES: usize = 32; tokio::task_local! { static REQUEST_ID: String; @@ -63,6 +71,8 @@ pub enum SentryConfigError { InvalidDsn, #[error("CRANK_SENTRY_DSN is not valid UTF-8")] InvalidEnvironmentEncoding, + #[error("critical error event budget cannot hold the required fields")] + EventBudgetTooSmall, } #[derive(Clone, Copy, Debug, Eq, PartialEq)] @@ -74,6 +84,13 @@ pub enum CriticalErrorCategory { } impl CriticalErrorCategory { + const ALL: [Self; 4] = [ + Self::Panic, + Self::Startup, + Self::Internal, + Self::DataIntegrity, + ]; + pub const fn as_str(self) -> &'static str { match self { Self::Panic => "panic", @@ -117,11 +134,14 @@ pub(crate) fn init_sentry( identity: &ServiceIdentity, limits: RedactionLimits, config: SentryConfig, -) -> Option { - let dsn = config.dsn?; +) -> Result, SentryConfigError> { + let Some(dsn) = config.dsn else { + return Ok(None); + }; + validate_critical_event_budget(identity, limits)?; let identity = identity.clone(); let options = client_options(identity, limits); - Some(sentry::init((dsn, options))) + Ok(Some(sentry::init((dsn, options)))) } fn client_options(identity: ServiceIdentity, limits: RedactionLimits) -> ClientOptions { @@ -181,7 +201,10 @@ fn sanitize_event( Event { event_id: event.event_id, level: Level::Error, - fingerprint: Cow::Owned(vec![Cow::Borrowed(category.as_str())]), + fingerprint: Cow::Owned(vec![ + Cow::Owned(identity.service().to_owned()), + Cow::Borrowed(category.as_str()), + ]), message: Some(CRITICAL_ERROR_MESSAGE.to_owned()), timestamp: event.timestamp, server_name: Some(Cow::Owned(identity.service().to_owned())), @@ -212,9 +235,44 @@ fn enforce_event_budget(mut event: Event<'static>, max_event_bytes: usize) -> Ev event.tags.remove("request_id"); event.tags.remove("trace_id"); + debug_assert!(serialized_event_len(&event) <= max_event_bytes); event } +fn validate_critical_event_budget( + identity: &ServiceIdentity, + limits: RedactionLimits, +) -> Result<(), SentryConfigError> { + if required_critical_event_budget(identity, limits) > limits.max_event_bytes { + return Err(SentryConfigError::EventBudgetTooSmall); + } + Ok(()) +} + +fn required_critical_event_budget(identity: &ServiceIdentity, limits: RedactionLimits) -> usize { + let unbounded_limits = RedactionLimits { + max_event_bytes: usize::MAX, + ..limits + }; + CriticalErrorCategory::ALL + .into_iter() + .map(|category| { + let event = sanitize_event( + Event { + tags: BTreeMap::from([("category".to_owned(), category.as_str().to_owned())]), + timestamp: SystemTime::UNIX_EPOCH, + ..Event::default() + }, + identity, + unbounded_limits, + ); + serialized_event_len(&event) + .saturating_add(MAX_SERIALIZED_TIMESTAMP_BYTES.saturating_sub(1)) + }) + .max() + .unwrap_or(usize::MAX) +} + fn serialized_event_len(event: &Event<'_>) -> usize { serde_json::to_vec(event).map_or(usize::MAX, |serialized| serialized.len()) } @@ -223,10 +281,12 @@ fn serialized_event_len(event: &Event<'_>) -> usize { mod tests { use std::{ collections::BTreeMap, + panic::AssertUnwindSafe, sync::{ Arc, atomic::{AtomicUsize, Ordering}, }, + time::Duration, }; use opentelemetry::trace::TracerProvider as _; @@ -237,7 +297,8 @@ mod tests { }; use super::{ - CriticalErrorCategory, capture_critical_error, client_options, sanitize_event, + CriticalErrorCategory, capture_critical_error, client_options, + required_critical_event_budget, sanitize_event, validate_critical_event_budget, with_request_correlation, }; use crate::{ @@ -395,6 +456,14 @@ mod tests { event.tags.get("category").map(String::as_str), Some("data_integrity") ); + assert_eq!( + event + .fingerprint + .iter() + .map(AsRef::as_ref) + .collect::>(), + vec!["admin-api", "data_integrity"] + ); assert_eq!( event.tags.get("request_id").map(String::as_str), Some("request-123") @@ -409,11 +478,33 @@ mod tests { fn panic_creates_exactly_one_sanitized_critical_event() { let options = sentry::apply_defaults(client_options(identity(), RedactionLimits::default())); + let runtime = tokio::runtime::Builder::new_current_thread() + .build() + .expect("runtime"); + let provider = SdkTracerProvider::builder().build(); + let tracer = provider.tracer("panic-correlation-test"); + let subscriber = build_subscriber_with_tracer( + ObservabilityConfig::new(identity(), "info", RedactionLimits::default()), + std::io::sink, + Some(tracer), + ) + .expect("subscriber"); + let dispatch = tracing::Dispatch::new(subscriber); let events = sentry::test::with_captured_events_options( || { - let result = std::panic::catch_unwind(|| { - panic!("password=must-not-leak"); - }); + let result = std::panic::catch_unwind(AssertUnwindSafe(|| { + tracing::dispatcher::with_default(&dispatch, || { + runtime.block_on(with_request_correlation( + "panic-request-123".to_owned(), + async { + let span = + tracing::info_span!(target: "crank::trace", "http.request"); + let _span_guard = span.enter(); + panic!("password=must-not-leak"); + }, + )); + }); + })); assert!(result.is_err()); }, options, @@ -425,24 +516,34 @@ mod tests { event.tags.get("category").map(String::as_str), Some("panic") ); + assert_eq!( + event.tags.get("request_id").map(String::as_str), + Some("panic-request-123") + ); + assert_eq!(event.tags.get("trace_id").map(String::len), Some(32)); let serialized = serde_json::to_string(event).expect("serialize event"); assert!(!serialized.contains("must-not-leak")); assert!(!serialized.contains("password")); + provider.shutdown().expect("provider shutdown"); } #[test] fn receiver_failure_does_not_change_product_result_or_recurse() { - struct DroppingTransport { + struct UnavailableTransport { attempts: AtomicUsize, } - impl sentry::Transport for DroppingTransport { + impl sentry::Transport for UnavailableTransport { fn send_envelope(&self, _envelope: Envelope) { self.attempts.fetch_add(1, Ordering::Relaxed); } + + fn flush(&self, _timeout: Duration) -> bool { + false + } } - let transport = Arc::new(DroppingTransport { + let transport = Arc::new(UnavailableTransport { attempts: AtomicUsize::new(0), }); let mut options = @@ -454,7 +555,10 @@ mod tests { ); options.transport = Some(Arc::new(transport.clone())); let client = Arc::new(sentry::Client::from(options)); - let hub = Arc::new(Hub::new(Some(client), Arc::new(Default::default()))); + let hub = Arc::new(Hub::new( + Some(Arc::clone(&client)), + Arc::new(Default::default()), + )); let product_result = Hub::run(hub, || { capture_critical_error(CriticalErrorCategory::Internal); @@ -463,5 +567,58 @@ mod tests { assert_eq!(product_result, 42); assert_eq!(transport.attempts.load(Ordering::Relaxed), 1); + assert!(!client.close(Some(Duration::from_millis(10)))); + assert_eq!(transport.attempts.load(Ordering::Relaxed), 1); + } + + #[test] + fn critical_event_budget_rejects_identity_that_does_not_fit() { + let maximum_label = "a".repeat(64); + let identity = + ServiceIdentity::try_new(maximum_label.clone(), maximum_label.clone(), maximum_label) + .expect("maximum identity"); + let limits = RedactionLimits { + max_event_bytes: 512, + ..RedactionLimits::default() + }; + + assert!(matches!( + validate_critical_event_budget(&identity, limits), + Err(super::SentryConfigError::EventBudgetTooSmall) + )); + validate_critical_event_budget( + &identity, + RedactionLimits { + max_event_bytes: 1024, + ..limits + }, + ) + .expect("larger budget must hold the required identity"); + } + + #[test] + fn critical_event_budget_covers_every_category() { + let identity = identity(); + let limits = RedactionLimits::default(); + let required_budget = required_critical_event_budget(&identity, limits); + + validate_critical_event_budget( + &identity, + RedactionLimits { + max_event_bytes: required_budget, + ..limits + }, + ) + .expect("exact required budget must be accepted"); + assert!(matches!( + validate_critical_event_budget( + &identity, + RedactionLimits { + max_event_bytes: required_budget - 1, + ..limits + }, + ), + Err(super::SentryConfigError::EventBudgetTooSmall) + )); } } diff --git a/crates/crank-observability/src/lifecycle.rs b/crates/crank-observability/src/lifecycle.rs index fca56c6..6004a2d 100644 --- a/crates/crank-observability/src/lifecycle.rs +++ b/crates/crank-observability/src/lifecycle.rs @@ -33,7 +33,7 @@ impl ObservabilityLifecycle { .map_err(|_| ObservabilityInitError::SubscriberAlreadyInitialized)?; let metrics_handle = install_prometheus_recorder(&identity)?; register_metric_schema(); - let sentry_guard = init_sentry(&identity, redaction_limits, sentry_config); + let sentry_guard = init_sentry(&identity, redaction_limits, sentry_config)?; Ok(Self { metrics_handle, diff --git a/docs/observability.md b/docs/observability.md index 1ae16e1..8ae0844 100644 --- a/docs/observability.md +++ b/docs/observability.md @@ -99,6 +99,11 @@ CRANK_SENTRY_DSN=https://public-key@errors.example.com/1 - доступные `request_id` и `trace_id`; - статическое сообщение без исходного текста ошибки. +События группируются по паре `service` и закрытой категории, поэтому одинаковая +ошибка `admin-api` и `mcp-server` не объединяется в один инцидент. Если +обязательные поля идентичности не помещаются в заданный предел события, запуск +останавливается безопасной типизированной ошибкой до приёма запросов. + До отправки удаляются request, user, breadcrumbs, URL, query, cookie, authorization, payload, произвольные contexts и extra, а также исходный текст panic или ошибки. Performance tracing, журналы, показатели и отслеживание diff --git a/docs/runtime-config.md b/docs/runtime-config.md index 299bb80..ab0af26 100644 --- a/docs/runtime-config.md +++ b/docs/runtime-config.md @@ -151,6 +151,11 @@ CRANK_LOG_LEVEL=info Внешний сборщик журналов не обязателен. Его отсутствие не влияет на `/health` и `/ready`. +`CRANK_INVOCATION_LOG_RETENTION_DAYS` задаёт срок хранения подробной истории +вызовов в днях. Допустимый диапазон: от `1` до `36500`, значение по умолчанию: +`30`. Значение вне диапазона останавливает запуск до создания фоновой задачи +очистки. + ## Критические ошибки - `CRANK_SENTRY_DSN` — DSN внешнего Sentry-совместимого приёмника.