исправить: закрыть ревью критических ошибок
CI / Rust Checks (pull_request) Successful in 6m4s
CI / UI Checks (pull_request) Successful in 5s
CI / Community Image Smoke (pull_request) Successful in 4m27s
CI / Frontend E2E (pull_request) Successful in 5m19s
CI / Deploy (pull_request) Has been skipped
CI / Rust Checks (push) Successful in 6m5s
CI / UI Checks (push) Successful in 5s
CI / Community Image Smoke (push) Successful in 1m5s
CI / Frontend E2E (push) Successful in 3m54s
CI / Deploy (push) Failing after 45s

This commit is contained in:
2026-07-31 09:31:38 +03:00
parent 9b1a739e39
commit c30461cc92
7 changed files with 200 additions and 27 deletions
+11 -10
View File
@@ -21,6 +21,8 @@ use sqlx::postgres::PgConnectOptions;
use tokio::net::TcpListener; use tokio::net::TcpListener;
use tracing::{info, warn}; use tracing::{info, warn};
const MAX_INVOCATION_LOG_RETENTION_DAYS: i64 = 36_500;
#[tokio::main] #[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> { async fn main() -> Result<(), Box<dyn std::error::Error>> {
let observability = crank_observability::init(ObservabilityConfig::from_env( let observability = crank_observability::init(ObservabilityConfig::from_env(
@@ -106,8 +108,7 @@ async fn run(
.with_outbound_http_policy(outbound_http_policy) .with_outbound_http_policy(outbound_http_policy)
.with_identity_provider(std::sync::Arc::new(identity_provider)) .with_identity_provider(std::sync::Arc::new(identity_provider))
.build(); .build();
let invocation_log_retention_days = let invocation_log_retention_days = invocation_log_retention_days_from_env()?;
positive_i64_from_env("CRANK_INVOCATION_LOG_RETENTION_DAYS", 30)?;
service.bootstrap_admin_user().await?; service.bootstrap_admin_user().await?;
if env_flag("CRANK_DEMO_SEED") { if env_flag("CRANK_DEMO_SEED") {
service.seed_demo_assets().await?; service.seed_demo_assets().await?;
@@ -159,17 +160,17 @@ async fn run(
Ok(()) Ok(())
} }
fn positive_i64_from_env( fn invocation_log_retention_days_from_env() -> Result<i64, Box<dyn std::error::Error>> {
name: &'static str, const NAME: &str = "CRANK_INVOCATION_LOG_RETENTION_DAYS";
default: i64, let value = match env::var(NAME) {
) -> Result<i64, Box<dyn std::error::Error>> {
let value = match env::var(name) {
Ok(raw) => raw.parse::<i64>()?, Ok(raw) => raw.parse::<i64>()?,
Err(env::VarError::NotPresent) => default, Err(env::VarError::NotPresent) => 30,
Err(error) => return Err(error.into()), Err(error) => return Err(error.into()),
}; };
if value <= 0 { if !(1..=MAX_INVOCATION_LOG_RETENTION_DAYS).contains(&value) {
return Err(format!("{name} must be greater than zero").into()); return Err(
format!("{NAME} must be between 1 and {MAX_INVOCATION_LOG_RETENTION_DAYS}").into(),
);
} }
Ok(value) Ok(value)
} }
+7 -3
View File
@@ -1,5 +1,5 @@
use axum::{ use axum::{
extract::Request, extract::{MatchedPath, Request},
http::{HeaderName, HeaderValue}, http::{HeaderName, HeaderValue},
middleware::Next, middleware::Next,
response::Response, response::Response,
@@ -19,7 +19,11 @@ pub async fn apply_request_context(mut request: Request, next: Next) -> Response
request_id: RequestId::resolve_from_headers(request.headers()).into_string(), request_id: RequestId::resolve_from_headers(request.headers()).into_string(),
}; };
let method = request.method().clone(); let method = request.method().clone();
let path = request.uri().path().to_owned(); let route = request
.extensions()
.get::<MatchedPath>()
.map_or("unmatched", MatchedPath::as_str)
.to_owned();
let span = info_span!( let span = info_span!(
target: "crank::trace", target: "crank::trace",
"http.request", "http.request",
@@ -34,7 +38,7 @@ pub async fn apply_request_context(mut request: Request, next: Next) -> Response
name: "admin.request.completed", name: "admin.request.completed",
request_id = %context.request_id, request_id = %context.request_id,
method = %method, method = %method,
path, route,
status = response.status().as_u16(), status = response.status().as_u16(),
"admin request completed" "admin request completed"
); );
@@ -65,6 +65,7 @@ async fn logs_request_completion_and_rejects_untrusted_values() {
.unwrap(); .unwrap();
assert_eq!(event["request_id"], "req_admin_trace_123"); assert_eq!(event["request_id"], "req_admin_trace_123");
assert_eq!(event["fields"]["status"], 200); assert_eq!(event["fields"]["status"], 200);
assert_eq!(event["fields"]["route"], "/probe");
let invalid_response = app let invalid_response = app
.oneshot( .oneshot(
+168 -11
View File
@@ -1,4 +1,10 @@
use std::{borrow::Cow, collections::BTreeMap, env, fmt, future::Future, time::Duration}; use std::{
borrow::Cow,
collections::BTreeMap,
env, fmt,
future::Future,
time::{Duration, SystemTime},
};
use sentry::{ use sentry::{
ClientInitGuard, ClientOptions, ClientInitGuard, ClientOptions,
@@ -14,6 +20,8 @@ use crate::{
const SENTRY_DSN_ENV: &str = "CRANK_SENTRY_DSN"; const SENTRY_DSN_ENV: &str = "CRANK_SENTRY_DSN";
const CRITICAL_ERROR_MESSAGE: &str = "critical error"; const CRITICAL_ERROR_MESSAGE: &str = "critical error";
const SENTRY_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(2); const SENTRY_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(2);
// Sentry serializes SystemTime as a finite f64; this keeps conservative fixed headroom.
const MAX_SERIALIZED_TIMESTAMP_BYTES: usize = 32;
tokio::task_local! { tokio::task_local! {
static REQUEST_ID: String; static REQUEST_ID: String;
@@ -63,6 +71,8 @@ pub enum SentryConfigError {
InvalidDsn, InvalidDsn,
#[error("CRANK_SENTRY_DSN is not valid UTF-8")] #[error("CRANK_SENTRY_DSN is not valid UTF-8")]
InvalidEnvironmentEncoding, InvalidEnvironmentEncoding,
#[error("critical error event budget cannot hold the required fields")]
EventBudgetTooSmall,
} }
#[derive(Clone, Copy, Debug, Eq, PartialEq)] #[derive(Clone, Copy, Debug, Eq, PartialEq)]
@@ -74,6 +84,13 @@ pub enum CriticalErrorCategory {
} }
impl CriticalErrorCategory { impl CriticalErrorCategory {
const ALL: [Self; 4] = [
Self::Panic,
Self::Startup,
Self::Internal,
Self::DataIntegrity,
];
pub const fn as_str(self) -> &'static str { pub const fn as_str(self) -> &'static str {
match self { match self {
Self::Panic => "panic", Self::Panic => "panic",
@@ -117,11 +134,14 @@ pub(crate) fn init_sentry(
identity: &ServiceIdentity, identity: &ServiceIdentity,
limits: RedactionLimits, limits: RedactionLimits,
config: SentryConfig, config: SentryConfig,
) -> Option<ClientInitGuard> { ) -> Result<Option<ClientInitGuard>, SentryConfigError> {
let dsn = config.dsn?; let Some(dsn) = config.dsn else {
return Ok(None);
};
validate_critical_event_budget(identity, limits)?;
let identity = identity.clone(); let identity = identity.clone();
let options = client_options(identity, limits); let options = client_options(identity, limits);
Some(sentry::init((dsn, options))) Ok(Some(sentry::init((dsn, options))))
} }
fn client_options(identity: ServiceIdentity, limits: RedactionLimits) -> ClientOptions { fn client_options(identity: ServiceIdentity, limits: RedactionLimits) -> ClientOptions {
@@ -181,7 +201,10 @@ fn sanitize_event(
Event { Event {
event_id: event.event_id, event_id: event.event_id,
level: Level::Error, level: Level::Error,
fingerprint: Cow::Owned(vec![Cow::Borrowed(category.as_str())]), fingerprint: Cow::Owned(vec![
Cow::Owned(identity.service().to_owned()),
Cow::Borrowed(category.as_str()),
]),
message: Some(CRITICAL_ERROR_MESSAGE.to_owned()), message: Some(CRITICAL_ERROR_MESSAGE.to_owned()),
timestamp: event.timestamp, timestamp: event.timestamp,
server_name: Some(Cow::Owned(identity.service().to_owned())), server_name: Some(Cow::Owned(identity.service().to_owned())),
@@ -212,9 +235,44 @@ fn enforce_event_budget(mut event: Event<'static>, max_event_bytes: usize) -> Ev
event.tags.remove("request_id"); event.tags.remove("request_id");
event.tags.remove("trace_id"); event.tags.remove("trace_id");
debug_assert!(serialized_event_len(&event) <= max_event_bytes);
event event
} }
fn validate_critical_event_budget(
identity: &ServiceIdentity,
limits: RedactionLimits,
) -> Result<(), SentryConfigError> {
if required_critical_event_budget(identity, limits) > limits.max_event_bytes {
return Err(SentryConfigError::EventBudgetTooSmall);
}
Ok(())
}
fn required_critical_event_budget(identity: &ServiceIdentity, limits: RedactionLimits) -> usize {
let unbounded_limits = RedactionLimits {
max_event_bytes: usize::MAX,
..limits
};
CriticalErrorCategory::ALL
.into_iter()
.map(|category| {
let event = sanitize_event(
Event {
tags: BTreeMap::from([("category".to_owned(), category.as_str().to_owned())]),
timestamp: SystemTime::UNIX_EPOCH,
..Event::default()
},
identity,
unbounded_limits,
);
serialized_event_len(&event)
.saturating_add(MAX_SERIALIZED_TIMESTAMP_BYTES.saturating_sub(1))
})
.max()
.unwrap_or(usize::MAX)
}
fn serialized_event_len(event: &Event<'_>) -> usize { fn serialized_event_len(event: &Event<'_>) -> usize {
serde_json::to_vec(event).map_or(usize::MAX, |serialized| serialized.len()) serde_json::to_vec(event).map_or(usize::MAX, |serialized| serialized.len())
} }
@@ -223,10 +281,12 @@ fn serialized_event_len(event: &Event<'_>) -> usize {
mod tests { mod tests {
use std::{ use std::{
collections::BTreeMap, collections::BTreeMap,
panic::AssertUnwindSafe,
sync::{ sync::{
Arc, Arc,
atomic::{AtomicUsize, Ordering}, atomic::{AtomicUsize, Ordering},
}, },
time::Duration,
}; };
use opentelemetry::trace::TracerProvider as _; use opentelemetry::trace::TracerProvider as _;
@@ -237,7 +297,8 @@ mod tests {
}; };
use super::{ use super::{
CriticalErrorCategory, capture_critical_error, client_options, sanitize_event, CriticalErrorCategory, capture_critical_error, client_options,
required_critical_event_budget, sanitize_event, validate_critical_event_budget,
with_request_correlation, with_request_correlation,
}; };
use crate::{ use crate::{
@@ -395,6 +456,14 @@ mod tests {
event.tags.get("category").map(String::as_str), event.tags.get("category").map(String::as_str),
Some("data_integrity") Some("data_integrity")
); );
assert_eq!(
event
.fingerprint
.iter()
.map(AsRef::as_ref)
.collect::<Vec<&str>>(),
vec!["admin-api", "data_integrity"]
);
assert_eq!( assert_eq!(
event.tags.get("request_id").map(String::as_str), event.tags.get("request_id").map(String::as_str),
Some("request-123") Some("request-123")
@@ -409,11 +478,33 @@ mod tests {
fn panic_creates_exactly_one_sanitized_critical_event() { fn panic_creates_exactly_one_sanitized_critical_event() {
let options = let options =
sentry::apply_defaults(client_options(identity(), RedactionLimits::default())); sentry::apply_defaults(client_options(identity(), RedactionLimits::default()));
let runtime = tokio::runtime::Builder::new_current_thread()
.build()
.expect("runtime");
let provider = SdkTracerProvider::builder().build();
let tracer = provider.tracer("panic-correlation-test");
let subscriber = build_subscriber_with_tracer(
ObservabilityConfig::new(identity(), "info", RedactionLimits::default()),
std::io::sink,
Some(tracer),
)
.expect("subscriber");
let dispatch = tracing::Dispatch::new(subscriber);
let events = sentry::test::with_captured_events_options( let events = sentry::test::with_captured_events_options(
|| { || {
let result = std::panic::catch_unwind(|| { let result = std::panic::catch_unwind(AssertUnwindSafe(|| {
tracing::dispatcher::with_default(&dispatch, || {
runtime.block_on(with_request_correlation(
"panic-request-123".to_owned(),
async {
let span =
tracing::info_span!(target: "crank::trace", "http.request");
let _span_guard = span.enter();
panic!("password=must-not-leak"); panic!("password=must-not-leak");
},
));
}); });
}));
assert!(result.is_err()); assert!(result.is_err());
}, },
options, options,
@@ -425,24 +516,34 @@ mod tests {
event.tags.get("category").map(String::as_str), event.tags.get("category").map(String::as_str),
Some("panic") Some("panic")
); );
assert_eq!(
event.tags.get("request_id").map(String::as_str),
Some("panic-request-123")
);
assert_eq!(event.tags.get("trace_id").map(String::len), Some(32));
let serialized = serde_json::to_string(event).expect("serialize event"); let serialized = serde_json::to_string(event).expect("serialize event");
assert!(!serialized.contains("must-not-leak")); assert!(!serialized.contains("must-not-leak"));
assert!(!serialized.contains("password")); assert!(!serialized.contains("password"));
provider.shutdown().expect("provider shutdown");
} }
#[test] #[test]
fn receiver_failure_does_not_change_product_result_or_recurse() { fn receiver_failure_does_not_change_product_result_or_recurse() {
struct DroppingTransport { struct UnavailableTransport {
attempts: AtomicUsize, attempts: AtomicUsize,
} }
impl sentry::Transport for DroppingTransport { impl sentry::Transport for UnavailableTransport {
fn send_envelope(&self, _envelope: Envelope) { fn send_envelope(&self, _envelope: Envelope) {
self.attempts.fetch_add(1, Ordering::Relaxed); self.attempts.fetch_add(1, Ordering::Relaxed);
} }
fn flush(&self, _timeout: Duration) -> bool {
false
}
} }
let transport = Arc::new(DroppingTransport { let transport = Arc::new(UnavailableTransport {
attempts: AtomicUsize::new(0), attempts: AtomicUsize::new(0),
}); });
let mut options = let mut options =
@@ -454,7 +555,10 @@ mod tests {
); );
options.transport = Some(Arc::new(transport.clone())); options.transport = Some(Arc::new(transport.clone()));
let client = Arc::new(sentry::Client::from(options)); let client = Arc::new(sentry::Client::from(options));
let hub = Arc::new(Hub::new(Some(client), Arc::new(Default::default()))); let hub = Arc::new(Hub::new(
Some(Arc::clone(&client)),
Arc::new(Default::default()),
));
let product_result = Hub::run(hub, || { let product_result = Hub::run(hub, || {
capture_critical_error(CriticalErrorCategory::Internal); capture_critical_error(CriticalErrorCategory::Internal);
@@ -463,5 +567,58 @@ mod tests {
assert_eq!(product_result, 42); assert_eq!(product_result, 42);
assert_eq!(transport.attempts.load(Ordering::Relaxed), 1); assert_eq!(transport.attempts.load(Ordering::Relaxed), 1);
assert!(!client.close(Some(Duration::from_millis(10))));
assert_eq!(transport.attempts.load(Ordering::Relaxed), 1);
}
#[test]
fn critical_event_budget_rejects_identity_that_does_not_fit() {
let maximum_label = "a".repeat(64);
let identity =
ServiceIdentity::try_new(maximum_label.clone(), maximum_label.clone(), maximum_label)
.expect("maximum identity");
let limits = RedactionLimits {
max_event_bytes: 512,
..RedactionLimits::default()
};
assert!(matches!(
validate_critical_event_budget(&identity, limits),
Err(super::SentryConfigError::EventBudgetTooSmall)
));
validate_critical_event_budget(
&identity,
RedactionLimits {
max_event_bytes: 1024,
..limits
},
)
.expect("larger budget must hold the required identity");
}
#[test]
fn critical_event_budget_covers_every_category() {
let identity = identity();
let limits = RedactionLimits::default();
let required_budget = required_critical_event_budget(&identity, limits);
validate_critical_event_budget(
&identity,
RedactionLimits {
max_event_bytes: required_budget,
..limits
},
)
.expect("exact required budget must be accepted");
assert!(matches!(
validate_critical_event_budget(
&identity,
RedactionLimits {
max_event_bytes: required_budget - 1,
..limits
},
),
Err(super::SentryConfigError::EventBudgetTooSmall)
));
} }
} }
+1 -1
View File
@@ -33,7 +33,7 @@ impl ObservabilityLifecycle {
.map_err(|_| ObservabilityInitError::SubscriberAlreadyInitialized)?; .map_err(|_| ObservabilityInitError::SubscriberAlreadyInitialized)?;
let metrics_handle = install_prometheus_recorder(&identity)?; let metrics_handle = install_prometheus_recorder(&identity)?;
register_metric_schema(); register_metric_schema();
let sentry_guard = init_sentry(&identity, redaction_limits, sentry_config); let sentry_guard = init_sentry(&identity, redaction_limits, sentry_config)?;
Ok(Self { Ok(Self {
metrics_handle, metrics_handle,
+5
View File
@@ -99,6 +99,11 @@ CRANK_SENTRY_DSN=https://public-key@errors.example.com/1
- доступные `request_id` и `trace_id`; - доступные `request_id` и `trace_id`;
- статическое сообщение без исходного текста ошибки. - статическое сообщение без исходного текста ошибки.
События группируются по паре `service` и закрытой категории, поэтому одинаковая
ошибка `admin-api` и `mcp-server` не объединяется в один инцидент. Если
обязательные поля идентичности не помещаются в заданный предел события, запуск
останавливается безопасной типизированной ошибкой до приёма запросов.
До отправки удаляются request, user, breadcrumbs, URL, query, cookie, До отправки удаляются request, user, breadcrumbs, URL, query, cookie,
authorization, payload, произвольные contexts и extra, а также исходный текст authorization, payload, произвольные contexts и extra, а также исходный текст
panic или ошибки. Performance tracing, журналы, показатели и отслеживание panic или ошибки. Performance tracing, журналы, показатели и отслеживание
+5
View File
@@ -151,6 +151,11 @@ CRANK_LOG_LEVEL=info
Внешний сборщик журналов не обязателен. Его отсутствие не влияет на `/health` Внешний сборщик журналов не обязателен. Его отсутствие не влияет на `/health`
и `/ready`. и `/ready`.
`CRANK_INVOCATION_LOG_RETENTION_DAYS` задаёт срок хранения подробной истории
вызовов в днях. Допустимый диапазон: от `1` до `36500`, значение по умолчанию:
`30`. Значение вне диапазона останавливает запуск до создания фоновой задачи
очистки.
## Критические ошибки ## Критические ошибки
- `CRANK_SENTRY_DSN` — DSN внешнего Sentry-совместимого приёмника. - `CRANK_SENTRY_DSN` — DSN внешнего Sentry-совместимого приёмника.