наблюдаемость: завершить базовый контур Community

Добавить структурированные журналы, метрики, трассировку и безопасный канал критических ошибок. Усилить границы рантайма, тесты, проверку зависимостей и сценарии развёртывания.

crates/crank-observability/src/error_reporting.rs

@@ -0,0 +1,467 @@
+use std::{borrow::Cow, collections::BTreeMap, env, fmt, future::Future, time::Duration};
+
+use sentry::{
+    ClientInitGuard, ClientOptions,
+    protocol::{Event, Level},
+    types::Dsn,
+};
+use thiserror::Error;
+
+use crate::{
+    RedactionLimits, ServiceIdentity, propagation::current_trace_id, redaction::truncate_string,
+};
+
+const SENTRY_DSN_ENV: &str = "CRANK_SENTRY_DSN";
+const CRITICAL_ERROR_MESSAGE: &str = "critical error";
+const SENTRY_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(2);
+
+tokio::task_local! {
+    static REQUEST_ID: String;
+}
+
+pub struct SentryConfig {
+    dsn: Option<Dsn>,
+}
+
+impl SentryConfig {
+    pub fn parse(value: Option<&str>) -> Result<Self, SentryConfigError> {
+        let Some(value) = value.map(str::trim).filter(|value| !value.is_empty()) else {
+            return Ok(Self { dsn: None });
+        };
+
+        let dsn = value
+            .parse::<Dsn>()
+            .map_err(|_| SentryConfigError::InvalidDsn)?;
+        Ok(Self { dsn: Some(dsn) })
+    }
+
+    pub fn from_env() -> Result<Self, SentryConfigError> {
+        match env::var(SENTRY_DSN_ENV) {
+            Ok(value) => Self::parse(Some(&value)),
+            Err(env::VarError::NotPresent) => Self::parse(None),
+            Err(env::VarError::NotUnicode(_)) => Err(SentryConfigError::InvalidEnvironmentEncoding),
+        }
+    }
+
+    pub fn enabled(&self) -> bool {
+        self.dsn.is_some()
+    }
+}
+
+impl fmt::Debug for SentryConfig {
+    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
+        formatter
+            .debug_struct("SentryConfig")
+            .field("enabled", &self.enabled())
+            .finish()
+    }
+}
+
+#[derive(Debug, Error)]
+pub enum SentryConfigError {
+    #[error("CRANK_SENTRY_DSN is not a valid Sentry DSN")]
+    InvalidDsn,
+    #[error("CRANK_SENTRY_DSN is not valid UTF-8")]
+    InvalidEnvironmentEncoding,
+}
+
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+pub enum CriticalErrorCategory {
+    Panic,
+    Startup,
+    Internal,
+    DataIntegrity,
+}
+
+impl CriticalErrorCategory {
+    pub const fn as_str(self) -> &'static str {
+        match self {
+            Self::Panic => "panic",
+            Self::Startup => "startup",
+            Self::Internal => "internal",
+            Self::DataIntegrity => "data_integrity",
+        }
+    }
+
+    fn parse(value: &str) -> Option<Self> {
+        match value {
+            "panic" => Some(Self::Panic),
+            "startup" => Some(Self::Startup),
+            "internal" => Some(Self::Internal),
+            "data_integrity" => Some(Self::DataIntegrity),
+            _ => None,
+        }
+    }
+}
+
+pub fn capture_critical_error(category: CriticalErrorCategory) {
+    let mut tags = correlation_tags();
+    tags.insert("category".to_owned(), category.as_str().to_owned());
+    sentry::capture_event(Event {
+        level: Level::Error,
+        message: Some(CRITICAL_ERROR_MESSAGE.to_owned()),
+        fingerprint: Cow::Owned(vec![Cow::Borrowed(category.as_str())]),
+        tags,
+        ..Event::default()
+    });
+}
+
+pub async fn with_request_correlation<F>(request_id: String, future: F) -> F::Output
+where
+    F: Future,
+{
+    REQUEST_ID.scope(request_id, future).await
+}
+
+pub(crate) fn init_sentry(
+    identity: &ServiceIdentity,
+    limits: RedactionLimits,
+    config: SentryConfig,
+) -> Option<ClientInitGuard> {
+    let dsn = config.dsn?;
+    let identity = identity.clone();
+    let options = client_options(identity, limits);
+    Some(sentry::init((dsn, options)))
+}
+
+fn client_options(identity: ServiceIdentity, limits: RedactionLimits) -> ClientOptions {
+    let release = identity.version().to_owned();
+    let environment = identity.environment().to_owned();
+    let service = identity.service().to_owned();
+    let sanitizer_identity = identity.clone();
+
+    let mut options = ClientOptions::default();
+    options.release = Some(Cow::Owned(release));
+    options.environment = Some(Cow::Owned(environment));
+    options.server_name = Some(Cow::Owned(service));
+    options.traces_sampling_strategy = sentry::TracesSamplingStrategy::Disabled;
+    options.max_breadcrumbs = 0;
+    options.attach_stacktrace = false;
+    options.send_default_pii = false;
+    options.before_send = Some(std::sync::Arc::new(move |event| {
+        Some(sanitize_event(event, &sanitizer_identity, limits))
+    }));
+    options.shutdown_timeout = SENTRY_SHUTDOWN_TIMEOUT;
+    options.auto_session_tracking = false;
+    options.enable_logs = false;
+    options.enable_metrics = false;
+    options
+}
+
+fn sanitize_event(
+    event: Event<'static>,
+    identity: &ServiceIdentity,
+    limits: RedactionLimits,
+) -> Event<'static> {
+    let category = event
+        .tags
+        .get("category")
+        .and_then(|value| CriticalErrorCategory::parse(value))
+        .unwrap_or_else(|| {
+            if event.exception.is_empty() {
+                CriticalErrorCategory::Internal
+            } else {
+                CriticalErrorCategory::Panic
+            }
+        });
+    let mut tags = correlation_tags()
+        .into_iter()
+        .map(|(key, value)| (key, truncate_string(&value, limits.max_string_bytes)))
+        .collect::<BTreeMap<_, _>>();
+    for key in ["request_id", "trace_id"] {
+        if let Some(value) = event.tags.get(key) {
+            tags.entry(key.to_owned())
+                .or_insert_with(|| truncate_string(value, limits.max_string_bytes));
+        }
+    }
+    tags.insert("service".to_owned(), identity.service().to_owned());
+    tags.insert("category".to_owned(), category.as_str().to_owned());
+
+    enforce_event_budget(
+        Event {
+            event_id: event.event_id,
+            level: Level::Error,
+            fingerprint: Cow::Owned(vec![Cow::Borrowed(category.as_str())]),
+            message: Some(CRITICAL_ERROR_MESSAGE.to_owned()),
+            timestamp: event.timestamp,
+            server_name: Some(Cow::Owned(identity.service().to_owned())),
+            release: Some(Cow::Owned(identity.version().to_owned())),
+            environment: Some(Cow::Owned(identity.environment().to_owned())),
+            tags,
+            ..Event::default()
+        },
+        limits.max_event_bytes,
+    )
+}
+
+fn correlation_tags() -> BTreeMap<String, String> {
+    let mut tags = BTreeMap::new();
+    if let Ok(request_id) = REQUEST_ID.try_with(Clone::clone) {
+        tags.insert("request_id".to_owned(), request_id);
+    }
+    if let Some(trace_id) = current_trace_id() {
+        tags.insert("trace_id".to_owned(), trace_id);
+    }
+    tags
+}
+
+fn enforce_event_budget(mut event: Event<'static>, max_event_bytes: usize) -> Event<'static> {
+    if serialized_event_len(&event) <= max_event_bytes {
+        return event;
+    }
+
+    event.tags.remove("request_id");
+    event.tags.remove("trace_id");
+    event
+}
+
+fn serialized_event_len(event: &Event<'_>) -> usize {
+    serde_json::to_vec(event).map_or(usize::MAX, |serialized| serialized.len())
+}
+
+#[cfg(test)]
+mod tests {
+    use std::{
+        collections::BTreeMap,
+        sync::{
+            Arc,
+            atomic::{AtomicUsize, Ordering},
+        },
+    };
+
+    use opentelemetry::trace::TracerProvider as _;
+    use opentelemetry_sdk::trace::SdkTracerProvider;
+    use sentry::{
+        Envelope, Hub,
+        protocol::{Breadcrumb, Context, Exception, Request, User, Value, Values},
+    };
+
+    use super::{
+        CriticalErrorCategory, capture_critical_error, client_options, sanitize_event,
+        with_request_correlation,
+    };
+    use crate::{
+        ObservabilityConfig, RedactionLimits, ServiceIdentity,
+        logging::build_subscriber_with_tracer,
+    };
+
+    fn identity() -> ServiceIdentity {
+        ServiceIdentity::try_new("admin-api", "1.2.3", "test").expect("valid identity")
+    }
+
+    #[test]
+    fn client_disables_non_error_telemetry() {
+        let options = client_options(identity(), RedactionLimits::default());
+
+        assert_eq!(options.max_breadcrumbs, 0);
+        assert!(!options.attach_stacktrace);
+        assert!(!options.send_default_pii);
+        assert!(!options.auto_session_tracking);
+        assert!(!options.enable_logs);
+        assert!(!options.enable_metrics);
+        assert!(options.before_send.is_some());
+    }
+
+    #[test]
+    fn sanitizer_uses_a_strict_allowlist() {
+        let mut tags = BTreeMap::new();
+        tags.insert("category".to_owned(), "data_integrity".to_owned());
+        tags.insert("secret".to_owned(), "must-not-leak".to_owned());
+        let mut contexts = BTreeMap::new();
+        contexts.insert(
+            "secret".to_owned(),
+            Context::Other(BTreeMap::from([(
+                "token".to_owned(),
+                Value::String("must-not-leak".to_owned()),
+            )])),
+        );
+        let event = sentry::protocol::Event {
+            message: Some("password=must-not-leak".to_owned()),
+            request: Some(Request::default()),
+            user: Some(User::default()),
+            breadcrumbs: Values {
+                values: vec![Breadcrumb::default()],
+            },
+            exception: Values {
+                values: vec![Exception {
+                    value: Some("must-not-leak".to_owned()),
+                    ..Exception::default()
+                }],
+            },
+            contexts,
+            extra: BTreeMap::from([(
+                "payload".to_owned(),
+                Value::String("must-not-leak".to_owned()),
+            )]),
+            tags,
+            ..sentry::protocol::Event::default()
+        };
+
+        let cleaned = sanitize_event(event, &identity(), RedactionLimits::default());
+        let serialized = serde_json::to_string(&cleaned).expect("serialize event");
+
+        assert_eq!(cleaned.message.as_deref(), Some("critical error"));
+        assert_eq!(
+            cleaned.tags.get("category").map(String::as_str),
+            Some(CriticalErrorCategory::DataIntegrity.as_str())
+        );
+        assert!(cleaned.request.is_none());
+        assert!(cleaned.user.is_none());
+        assert!(cleaned.breadcrumbs.is_empty());
+        assert!(cleaned.exception.is_empty());
+        assert!(cleaned.contexts.is_empty());
+        assert!(cleaned.extra.is_empty());
+        assert!(!serialized.contains("must-not-leak"));
+        assert!(!serialized.contains("password"));
+    }
+
+    #[test]
+    fn sanitizer_honours_total_event_budget() {
+        let limits = RedactionLimits {
+            max_string_bytes: 8 * 1024,
+            max_event_bytes: 512,
+            ..RedactionLimits::default()
+        };
+        let event = sentry::protocol::Event {
+            tags: BTreeMap::from([
+                ("category".to_owned(), "internal".to_owned()),
+                ("request_id".to_owned(), "r".repeat(8 * 1024)),
+                ("trace_id".to_owned(), "t".repeat(8 * 1024)),
+            ]),
+            ..sentry::protocol::Event::default()
+        };
+
+        let cleaned = sanitize_event(event, &identity(), limits);
+        let serialized = serde_json::to_vec(&cleaned).expect("serialize event");
+
+        assert!(serialized.len() <= limits.max_event_bytes);
+        assert_eq!(
+            cleaned.tags.get("category").map(String::as_str),
+            Some("internal")
+        );
+        assert_eq!(
+            cleaned.tags.get("service").map(String::as_str),
+            Some("admin-api")
+        );
+        assert!(!cleaned.tags.contains_key("request_id"));
+        assert!(!cleaned.tags.contains_key("trace_id"));
+    }
+
+    #[test]
+    fn expected_application_errors_do_not_create_critical_events() {
+        let options =
+            sentry::apply_defaults(client_options(identity(), RedactionLimits::default()));
+        let events = sentry::test::with_captured_events_options(
+            || tracing::error!("ordinary product error"),
+            options,
+        );
+
+        assert!(events.is_empty());
+    }
+
+    #[test]
+    fn explicit_critical_error_is_correlated_and_sanitized() {
+        let options =
+            sentry::apply_defaults(client_options(identity(), RedactionLimits::default()));
+        let runtime = tokio::runtime::Builder::new_current_thread()
+            .build()
+            .expect("runtime");
+        let provider = SdkTracerProvider::builder().build();
+        let tracer = provider.tracer("critical-error-test");
+        let subscriber = build_subscriber_with_tracer(
+            ObservabilityConfig::new(identity(), "info", RedactionLimits::default()),
+            std::io::sink,
+            Some(tracer),
+        )
+        .expect("subscriber");
+        let dispatch = tracing::Dispatch::new(subscriber);
+        let events = sentry::test::with_captured_events_options(
+            || {
+                tracing::dispatcher::with_default(&dispatch, || {
+                    runtime.block_on(with_request_correlation("request-123".to_owned(), async {
+                        let span = tracing::info_span!(target: "crank::trace", "http.request");
+                        let _span_guard = span.enter();
+                        capture_critical_error(CriticalErrorCategory::DataIntegrity);
+                    }));
+                });
+            },
+            options,
+        );
+
+        assert_eq!(events.len(), 1);
+        let event = &events[0];
+        assert_eq!(event.message.as_deref(), Some("critical error"));
+        assert_eq!(
+            event.tags.get("category").map(String::as_str),
+            Some("data_integrity")
+        );
+        assert_eq!(
+            event.tags.get("request_id").map(String::as_str),
+            Some("request-123")
+        );
+        assert_eq!(event.tags.get("trace_id").map(String::len), Some(32));
+        assert_eq!(event.release.as_deref(), Some("1.2.3"));
+        assert_eq!(event.environment.as_deref(), Some("test"));
+        assert_eq!(event.server_name.as_deref(), Some("admin-api"));
+    }
+
+    #[test]
+    fn panic_creates_exactly_one_sanitized_critical_event() {
+        let options =
+            sentry::apply_defaults(client_options(identity(), RedactionLimits::default()));
+        let events = sentry::test::with_captured_events_options(
+            || {
+                let result = std::panic::catch_unwind(|| {
+                    panic!("password=must-not-leak");
+                });
+                assert!(result.is_err());
+            },
+            options,
+        );
+
+        assert_eq!(events.len(), 1);
+        let event = &events[0];
+        assert_eq!(
+            event.tags.get("category").map(String::as_str),
+            Some("panic")
+        );
+        let serialized = serde_json::to_string(event).expect("serialize event");
+        assert!(!serialized.contains("must-not-leak"));
+        assert!(!serialized.contains("password"));
+    }
+
+    #[test]
+    fn receiver_failure_does_not_change_product_result_or_recurse() {
+        struct DroppingTransport {
+            attempts: AtomicUsize,
+        }
+
+        impl sentry::Transport for DroppingTransport {
+            fn send_envelope(&self, _envelope: Envelope) {
+                self.attempts.fetch_add(1, Ordering::Relaxed);
+            }
+        }
+
+        let transport = Arc::new(DroppingTransport {
+            attempts: AtomicUsize::new(0),
+        });
+        let mut options =
+            sentry::apply_defaults(client_options(identity(), RedactionLimits::default()));
+        options.dsn = Some(
+            "https://public@example.invalid/1"
+                .parse()
+                .expect("valid test DSN"),
+        );
+        options.transport = Some(Arc::new(transport.clone()));
+        let client = Arc::new(sentry::Client::from(options));
+        let hub = Arc::new(Hub::new(Some(client), Arc::new(Default::default())));
+
+        let product_result = Hub::run(hub, || {
+            capture_critical_error(CriticalErrorCategory::Internal);
+            42
+        });
+
+        assert_eq!(product_result, 42);
+        assert_eq!(transport.attempts.load(Ordering::Relaxed), 1);
+    }
+}